Merge branch 'master' into patch-6

research/tcn/train.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Trains TCN models (and baseline comparisons)."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from estimators.get_estimator import get_estimator
+from utils import util
+import tensorflow as tf
+tf.logging.set_verbosity(tf.logging.INFO)
+
+tf.flags.DEFINE_string(
+    'config_paths', '',
+    """
+    Path to a YAML configuration files defining FLAG values. Multiple files
+    can be separated by the `#` symbol. Files are merged recursively. Setting
+    a key in these files is equivalent to setting the FLAG value with
+    the same name.
+    """)
+tf.flags.DEFINE_string(
+    'model_params', '{}', 'YAML configuration string for the model parameters.')
+tf.app.flags.DEFINE_string('master', 'local',
+                           'BNS name of the TensorFlow master to use')
+tf.app.flags.DEFINE_string(
+    'logdir', '/tmp/tcn', 'Directory where to write event logs.')
+tf.app.flags.DEFINE_integer(
+    'task', 0, 'Task id of the replica running the training.')
+tf.app.flags.DEFINE_integer(
+    'ps_tasks', 0, 'Number of tasks in the ps job. If 0 no ps job is used.')
+FLAGS = tf.app.flags.FLAGS
+
+
+def main(_):
+  """Runs main training loop."""
+  # Parse config dict from yaml config files / command line flags.
+  config = util.ParseConfigsToLuaTable(
+      FLAGS.config_paths, FLAGS.model_params, save=True, logdir=FLAGS.logdir)
+
+  # Choose an estimator based on training strategy.
+  estimator = get_estimator(config, FLAGS.logdir)
+
+  # Run training
+  estimator.train()
+
+if __name__ == '__main__':
+  tf.app.run()

research/tcn/utils/luatables.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# pylint: disable=line-too-long,g-explicit-length-test
+"""A convenience class replicating some lua table syntax with a python dict.
+
+In general, should behave like a dictionary except that we can use dot notation
+ to access keys. Users should be careful to only provide keys suitable for
+ instance variable names.
+
+Nota bene: do not use the key "keys" since it will collide with the method keys.
+
+Usage example:
+
+>>> t = T(a=5,b='kaw', c=T(v=[],x=33))
+>>> t.a
+5
+>>> t.z = None
+>>> print t
+T(a=5, z=None, c=T(x=33, v=[]), b='kaw')
+
+>>> t2 = T({'h':'f','x':4})
+>>> t2
+T(h='f', x=4)
+>>> t2['x']
+4
+"""
+
+
+class T(object):
+  """Class for emulating lua tables."""
+
+  def __init__(self, *args, **kwargs):
+    if len(args) > 1 or (len(args) == 1 and len(kwargs) > 0):
+      errmsg = '''constructor only allows a single dict as a positional
+      argument or keyword arguments'''
+      raise ValueError(errmsg)
+    if len(args) == 1 and isinstance(args[0], dict):
+      self.__dict__.update(args[0])
+    else:
+      self.__dict__.update(kwargs)
+
+  def __repr__(self):
+    fmt = ', '.join('%s=%s' for i in range(len(self.__dict__)))
+    kwargstr = fmt % tuple(
+        x for tup in self.__dict__.items() for x in [str(tup[0]), repr(tup[1])])
+    return 'T(' + kwargstr + ')'
+
+  def __getitem__(self, key):
+    return self.__dict__[key]
+
+  def __setitem__(self, key, val):
+    self.__dict__[key] = val
+
+  def __delitem__(self, key):
+    del self.__dict__[key]
+
+  def __iter__(self):
+    return iter(self.__dict__)
+
+  def __len__(self):
+    return len(self.__dict__)
+
+  def keys(self):  # Needed for dict(T( ... )) to work.
+    return self.__dict__.keys()
+
+  def iteritems(self):
+    return self.__dict__.iteritems()

research/tcn/utils/progress.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""A utility class for reporting processing progress."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import datetime
+
+
+class Progress(object):
+  """A utility class for reporting processing progress."""
+
+  def __init__(self, target_size):
+    self.target_size = target_size
+    self.current_size = 0
+    self.start_time = datetime.datetime.now()
+
+  def Update(self, current_size):
+    """Replaces internal current_size with current_size."""
+    self.current_size = current_size
+
+  def Add(self, size):
+    """Increments internal current_size by size."""
+    self.current_size += size
+
+  def __str__(self):
+    processed = 1e-5 + self.current_size / float(self.target_size)
+    current_time = datetime.datetime.now()
+    elapsed = current_time - self.start_time
+    eta = datetime.timedelta(
+        seconds=elapsed.total_seconds() / processed - elapsed.total_seconds())
+    return "%d / %d (elapsed %s eta %s)" % (
+        self.current_size, self.target_size,
+        str(elapsed).split(".")[0],
+        str(eta).split(".")[0])

research/tcn/utils/util.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""General utility functions."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import numpy as np
+import six
+from utils.luatables import T
+import tensorflow as tf
+import yaml
+from yaml.constructor import ConstructorError
+# pylint: disable=invalid-name
+
+
+def GetFilesRecursively(topdir):
+  """Gets all records recursively for some topdir.
+
+  Args:
+    topdir: String, path to top directory.
+  Returns:
+    allpaths: List of Strings, full paths to all leaf records.
+  Raises:
+    ValueError: If there are no files found for this directory.
+  """
+  assert topdir
+  topdir = os.path.expanduser(topdir)
+  allpaths = []
+  for path, _, leaffiles in tf.gfile.Walk(topdir):
+    if leaffiles:
+      allpaths.extend([os.path.join(path, i) for i in leaffiles])
+  if not allpaths:
+    raise ValueError('No files found for top directory %s' % topdir)
+  return allpaths
+
+
+def NoDuplicatesConstructor(loader, node, deep=False):
+  """Check for duplicate keys."""
+  mapping = {}
+  for key_node, value_node in node.value:
+    key = loader.construct_object(key_node, deep=deep)
+    value = loader.construct_object(value_node, deep=deep)
+    if key in mapping:
+      raise ConstructorError('while constructing a mapping', node.start_mark,
+                             'found duplicate key (%s)' % key,
+                             key_node.start_mark)
+    mapping[key] = value
+  return loader.construct_mapping(node, deep)
+
+
+def WriteConfigAsYaml(config, logdir, filename):
+  """Writes a config dict as yaml to logdir/experiment.yml."""
+  if not tf.gfile.Exists(logdir):
+    tf.gfile.MakeDirs(logdir)
+  config_filename = os.path.join(logdir, filename)
+  with tf.gfile.GFile(config_filename, 'w') as f:
+    f.write(yaml.dump(config))
+  tf.logging.info('wrote config to %s', config_filename)
+
+
+def LoadConfigDict(config_paths, model_params):
+  """Loads config dictionary from specified yaml files or command line yaml."""
+
+  # Ensure that no duplicate keys can be loaded (causing pain).
+  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+                       NoDuplicatesConstructor)
+
+  # Handle either ',' or '#' separated config lists, since borg will only
+  # accept '#'.
+  sep = ',' if ',' in config_paths else '#'
+
+  # Load flags from config file.
+  final_config = {}
+  if config_paths:
+    for config_path in config_paths.split(sep):
+      config_path = config_path.strip()
+      if not config_path:
+        continue
+      config_path = os.path.abspath(config_path)
+      tf.logging.info('Loading config from %s', config_path)
+      with tf.gfile.GFile(config_path.strip()) as config_file:
+        config_flags = yaml.load(config_file)
+        final_config = DeepMergeDict(final_config, config_flags)
+  if model_params:
+    model_params = MaybeLoadYaml(model_params)
+    final_config = DeepMergeDict(final_config, model_params)
+  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
+  return final_config
+
+
+def MaybeLoadYaml(item):
+  """Parses item if it's a string. If it's a dictionary it's returned as-is."""
+  if isinstance(item, six.string_types):
+    return yaml.load(item)
+  elif isinstance(item, dict):
+    return item
+  else:
+    raise ValueError('Got {}, expected YAML string or dict', type(item))
+
+
+def DeepMergeDict(dict_x, dict_y, path=None):
+  """Recursively merges dict_y into dict_x."""
+  if path is None: path = []
+  for key in dict_y:
+    if key in dict_x:
+      if isinstance(dict_x[key], dict) and isinstance(dict_y[key], dict):
+        DeepMergeDict(dict_x[key], dict_y[key], path + [str(key)])
+      elif dict_x[key] == dict_y[key]:
+        pass  # same leaf value
+      else:
+        dict_x[key] = dict_y[key]
+    else:
+      dict_x[key] = dict_y[key]
+  return dict_x
+
+
+def ParseConfigsToLuaTable(config_paths, extra_model_params=None,
+                           save=False, save_name='final_training_config.yml',
+                           logdir=None):
+  """Maps config_paths and extra_model_params to a Luatable-like object."""
+  # Parse config dict from yaml config files / command line flags.
+  config = LoadConfigDict(config_paths, extra_model_params)
+  if save:
+    WriteConfigAsYaml(config, logdir, save_name)
+  # Convert config dictionary to T object with dot notation.
+  config = RecursivelyConvertToLuatable(config)
+  return config
+
+
+def SetNestedValue(d, keys, value):
+  """Sets a value in a nested dictionary.
+
+  Example:
+    d = {}, keys = ['data','augmentation','minscale'], value = 1.0.
+    returns {'data': {'augmentation' : {'minscale': 1.0 }}}
+
+  Args:
+    d: A dictionary to set a nested value in.
+    keys: list of dict keys nesting left to right.
+    value: the nested value to set.
+  Returns:
+    None
+  """
+  for key in keys[:-1]:
+    d = d.setdefault(key, {})
+  d[keys[-1]] = value
+
+
+def RecursivelyConvertToLuatable(yaml_dict):
+  """Converts a dictionary to a LuaTable-like T object."""
+  if isinstance(yaml_dict, dict):
+    yaml_dict = T(yaml_dict)
+  for key, item in yaml_dict.iteritems():
+    if isinstance(item, dict):
+      yaml_dict[key] = RecursivelyConvertToLuatable(item)
+  return yaml_dict
+
+
+def KNNIds(query_vec, target_seq, k=1):
+  """Gets the knn ids to the query vec from the target sequence."""
+  sorted_distances = KNNIdsWithDistances(query_vec, target_seq, k)
+  return [i[0] for i in sorted_distances]
+
+
+def KNNIdsWithDistances(query_vec, target_seq, k=1):
+  """Gets the knn ids to the query vec from the target sequence."""
+  if not isinstance(np.array(target_seq), np.ndarray):
+    target_seq = np.array(target_seq)
+  assert np.shape(query_vec) == np.shape(target_seq[0])
+  distances = [(i, np.linalg.norm(query_vec-target_vec)) for (
+      i, target_vec) in enumerate(target_seq)]
+  sorted_distances = sorted(distances, key=lambda x: x[1])
+  return sorted_distances[:k]
+
+
+def CopyLocalConfigsToCNS(outdir, configs, gfs_user):
+  """Copies experiment yaml config files to the job_logdir on /cns."""
+  assert configs
+  assert outdir
+  conf_files = configs.split(',')
+  for conf_file in conf_files:
+    copy_command = 'fileutil --gfs_user %s cp -f %s %s' % (
+        gfs_user, conf_file, outdir)
+    tf.logging.info(copy_command)
+    os.system(copy_command)
+
+
+def pairwise_distances(feature, squared=True):
+  """Computes the pairwise distance matrix in numpy.
+
+  Args:
+    feature: 2-D numpy array of size [number of data, feature dimension]
+    squared: Boolean. If true, output is the pairwise squared euclidean
+      distance matrix; else, output is the pairwise euclidean distance matrix.
+
+  Returns:
+    pdists: 2-D numpy array of size
+      [number of data, number of data].
+  """
+  triu = np.triu_indices(feature.shape[0], 1)
+  upper_tri_pdists = np.linalg.norm(feature[triu[1]] - feature[triu[0]], axis=1)
+  if squared:
+    upper_tri_pdists **= 2.
+  num_data = feature.shape[0]
+  pdists = np.zeros((num_data, num_data))
+  pdists[np.triu_indices(num_data, 1)] = upper_tri_pdists
+  # Make symmetrical.
+  pdists = pdists + pdists.T - np.diag(
+      pdists.diagonal())
+  return pdists
+
+
+def is_tfrecord_input(inp):
+  """Checks if input is a TFRecord or list of TFRecords."""
+  def _is_tfrecord(inp):
+    if not isinstance(inp, str):
+      return False
+    _, extension = os.path.splitext(inp)
+    return extension == '.tfrecord'
+  if isinstance(inp, str):
+    return _is_tfrecord(inp)
+  if isinstance(inp, list):
+    return all(map(_is_tfrecord, inp))
+  return False
+
+
+def is_np_array(inp):
+  if isinstance(inp, np.ndarray):
+    return True
+  if isinstance(inp, list):
+    return all([isinstance(i, np.ndarray) for i in inp])
+  return False

research/tcn/visualize_embeddings.py

+# Copyright 2017 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+r"""Visualizes embeddings in tensorboard.
+
+Usage:
+root=experimental/users/sermanet/imitation/mirror && \
+blaze build -c opt --copt=-mavx --config=cuda $root:visualize_embeddings && \
+blaze-bin/$root/visualize_embeddings \
+--checkpointdir $checkpointdir \
+--checkpoint_iter $checkpoint_iter \
+--embedding_records $embedding_records \
+--outdir $outdir \
+--num_embed 1000 \
+--sprite_dim 64 \
+--config_paths $configs \
+--logtostderr
+
+blaze build third_party/tensorboard && \
+blaze-bin/third_party/tensorboard/tensorboard --logdir=$outdir
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import random
+import cv2
+import numpy as np
+from scipy.misc import imresize
+from scipy.misc import imsave
+from estimators.get_estimator import get_estimator
+from utils import util
+import tensorflow as tf
+from tensorflow.contrib.tensorboard.plugins import projector
+tf.logging.set_verbosity(tf.logging.INFO)
+
+tf.flags.DEFINE_string(
+    'config_paths', '',
+    """
+    Path to a YAML configuration files defining FLAG values. Multiple files
+    can be separated by the `#` symbol. Files are merged recursively. Setting
+    a key in these files is equivalent to setting the FLAG value with
+    the same name.
+    """)
+tf.flags.DEFINE_string(
+    'model_params', '{}', 'YAML configuration string for the model parameters.')
+tf.app.flags.DEFINE_string(
+    'checkpoint_iter', '', 'Evaluate this specific checkpoint.')
+tf.app.flags.DEFINE_string(
+    'checkpointdir', '/tmp/tcn', 'Path to model checkpoints.')
+tf.app.flags.DEFINE_string(
+    'outdir', '/tmp/tcn', 'Path to write tensorboard info to.')
+tf.app.flags.DEFINE_integer(
+    'num_embed', 4000, 'Number of embeddings.')
+tf.app.flags.DEFINE_integer(
+    'num_sequences', -1, 'Number of sequences, -1 for all.')
+tf.app.flags.DEFINE_integer(
+    'sprite_dim', 64, 'Height, width of the square sprite image.')
+tf.app.flags.DEFINE_string(
+    'embedding_records', None, 'path to embedding records')
+FLAGS = tf.app.flags.FLAGS
+
+
+def images_to_sprite(data):
+  """Creates the sprite image along with any necessary padding.
+
+  Taken from: https://github.com/tensorflow/tensorflow/issues/6322
+
+  Args:
+    data: NxHxW[x3] tensor containing the images.
+
+  Returns:
+    data: Properly shaped HxWx3 image with any necessary padding.
+  """
+  if len(data.shape) == 3:
+    data = np.tile(data[..., np.newaxis], (1, 1, 1, 3))
+  data = data.astype(np.float32)
+  min_v = np.min(data.reshape((data.shape[0], -1)), axis=1)
+  data = (data.transpose(1, 2, 3, 0) - min_v).transpose(3, 0, 1, 2)
+  max_v = np.max(data.reshape((data.shape[0], -1)), axis=1)
+  data = (data.transpose(1, 2, 3, 0) / max_v).transpose(3, 0, 1, 2)
+  n = int(np.ceil(np.sqrt(data.shape[0])))
+  padding = ((0, n ** 2 - data.shape[0]), (0, 0),
+             (0, 0)) + ((0, 0),) * (data.ndim - 3)
+  data = np.pad(data, padding, mode='constant',
+                constant_values=0)
+  # Tile the individual thumbnails into an image.
+  data = data.reshape((n, n) + data.shape[1:]).transpose(
+      (0, 2, 1, 3) + tuple(range(4, data.ndim + 1)))
+  data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:])
+  data = (data * 255).astype(np.uint8)
+  return data
+
+
+def main(_):
+  """Runs main labeled eval loop."""
+  # Parse config dict from yaml config files / command line flags.
+  config = util.ParseConfigsToLuaTable(FLAGS.config_paths, FLAGS.model_params)
+
+  # Choose an estimator based on training strategy.
+  checkpointdir = FLAGS.checkpointdir
+  checkpoint_path = os.path.join(
+      '%s/model.ckpt-%s' % (checkpointdir, FLAGS.checkpoint_iter))
+  estimator = get_estimator(config, checkpointdir)
+
+  # Get records to embed.
+  validation_dir = FLAGS.embedding_records
+  validation_records = util.GetFilesRecursively(validation_dir)
+
+  sequences_to_data = {}
+  for (view_embeddings, view_raw_image_strings, seqname) in estimator.inference(
+      validation_records, checkpoint_path, config.data.embed_batch_size,
+      num_sequences=FLAGS.num_sequences):
+    sequences_to_data[seqname] = {
+        'embeddings': view_embeddings,
+        'images': view_raw_image_strings,
+    }
+
+  all_embeddings = np.zeros((0, config.embedding_size))
+  all_ims = []
+  all_seqnames = []
+
+  num_embeddings = FLAGS.num_embed
+  # Concatenate all views from all sequences into a big flat list.
+  for seqname, data in sequences_to_data.iteritems():
+    embs = data['embeddings']
+    ims = data['images']
+    for v in range(config.data.num_views):
+      for (emb, im) in zip(embs[v], ims[v]):
+        all_embeddings = np.append(all_embeddings, [emb], axis=0)
+        all_ims.append(im)
+        all_seqnames.append(seqname)
+
+  # Choose N indices uniformly from all images.
+  random_indices = range(all_embeddings.shape[0])
+  random.shuffle(random_indices)
+  viz_indices = random_indices[:num_embeddings]
+
+  # Extract embs.
+  viz_embs = np.array(all_embeddings[viz_indices])
+
+  # Extract and decode ims.
+  viz_ims = list(np.array(all_ims)[viz_indices])
+  decoded_ims = []
+
+  sprite_dim = FLAGS.sprite_dim
+  for i, im in enumerate(viz_ims):
+    if i % 100 == 0:
+      print('Decoding image %d/%d.' % (i, num_embeddings))
+    nparr_i = np.fromstring(str(im), np.uint8)
+    img_np = cv2.imdecode(nparr_i, 1)
+    img_np = img_np[..., [2, 1, 0]]
+
+    img_np = imresize(img_np, [sprite_dim, sprite_dim, 3])
+    decoded_ims.append(img_np)
+  decoded_ims = np.array(decoded_ims)
+
+  # Extract sequence names.
+  outdir = FLAGS.outdir
+
+  # The embedding variable, which needs to be stored
+  # Note this must a Variable not a Tensor!
+  embedding_var = tf.Variable(viz_embs, name='viz_embs')
+
+  with tf.Session() as sess:
+    sess.run(embedding_var.initializer)
+    summary_writer = tf.summary.FileWriter(outdir)
+    config = projector.ProjectorConfig()
+    embedding = config.embeddings.add()
+    embedding.tensor_name = embedding_var.name
+
+    # Comment out if you don't want sprites
+    embedding.sprite.image_path = os.path.join(outdir, 'sprite.png')
+    embedding.sprite.single_image_dim.extend(
+        [decoded_ims.shape[1], decoded_ims.shape[1]])
+
+    projector.visualize_embeddings(summary_writer, config)
+    saver = tf.train.Saver([embedding_var])
+    saver.save(sess, os.path.join(outdir, 'model2.ckpt'), 1)
+
+  sprite = images_to_sprite(decoded_ims)
+  imsave(os.path.join(outdir, 'sprite.png'), sprite)
+
+if __name__ == '__main__':
+  tf.app.run(main)

research/textsum/README.md

@@ -117,7 +117,7 @@ $ bazel-bin/textsum/seq2seq_attention \
     --log_root=textsum/log_root \
     --eval_dir=textsum/log_root/eval
 
-# Run the decode. Run it when the most is mostly converged.
+# Run the decode. Run it when the model is mostly converged.
 $ bazel-bin/textsum/seq2seq_attention \
     --mode=decode \
     --article_key=article \

samples/core/get_started/custom_estimator.py

@@ -17,67 +17,15 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import pandas as pd
 import tensorflow as tf
 
+import iris_data
+
 parser = argparse.ArgumentParser()
 parser.add_argument('--batch_size', default=100, type=int, help='batch size')
 parser.add_argument('--train_steps', default=1000, type=int,
                     help='number of training steps')
 
-TRAIN_URL = "http://download.tensorflow.org/data/iris_training.csv"
-TEST_URL = "http://download.tensorflow.org/data/iris_test.csv"
-
-CSV_COLUMN_NAMES = ['SepalLength', 'SepalWidth',
-                    'PetalLength', 'PetalWidth', 'Species']
-SPECIES = ['Sentosa', 'Versicolor', 'Virginica']
-
-
-def load_data(y_name='Species'):
-    """Returns the iris dataset as (train_x, train_y), (test_x, test_y)."""
-    train_path = tf.keras.utils.get_file(TRAIN_URL.split('/')[-1], TRAIN_URL)
-    train = pd.read_csv(train_path, names=CSV_COLUMN_NAMES, header=0)
-    train_x, train_y = train, train.pop(y_name)
-
-    test_path = tf.keras.utils.get_file(TEST_URL.split('/')[-1], TEST_URL)
-    test = pd.read_csv(test_path, names=CSV_COLUMN_NAMES, header=0)
-    test_x, test_y = test, test.pop(y_name)
-
-    return (train_x, train_y), (test_x, test_y)
-
-
-
-def train_input_fn(features, labels, batch_size):
-    """An input function for training"""
-    # Convert the inputs to a Dataset.
-    dataset = tf.data.Dataset.from_tensor_slices((features, labels))
-
-    # Shuffle, repeat, and batch the examples.
-    dataset = dataset.shuffle(1000).repeat().batch(batch_size)
-
-    # Return the read end of the pipeline.
-    return dataset.make_one_shot_iterator().get_next()
-
-
-def eval_input_fn(features, labels=None, batch_size=None):
-    """An input function for evaluation or prediction"""
-    if labels is None:
-        # No labels, use only features.
-        inputs = features
-    else:
-        inputs = (features, labels)
-
-    # Convert the inputs to a Dataset.
-    dataset = tf.data.Dataset.from_tensor_slices(inputs)
-
-    # Batch the examples
-    assert batch_size is not None, "batch_size must not be None"
-    dataset = dataset.batch(batch_size)
-
-    # Return the read end of the pipeline.
-    return dataset.make_one_shot_iterator().get_next()
-
-
 def my_model(features, labels, mode, params):
     """DNN with three hidden layers, and dropout of 0.1 probability."""
     # Create three fully connected layers each layer having a dropout
@@ -99,12 +47,8 @@ def my_model(features, labels, mode, params):
         }
         return tf.estimator.EstimatorSpec(mode, predictions=predictions)
 
-    # Convert the labels to a one-hot tensor of shape (length of features, 3)
-    # and with a on-value of 1 for each one-hot vector of length 3.
-    onehot_labels = tf.one_hot(labels, 3, 1, 0)
     # Compute loss.
-    loss = tf.losses.softmax_cross_entropy(
-        onehot_labels=onehot_labels, logits=logits)
+    loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
 
     # Compute evaluation metrics.
     accuracy = tf.metrics.accuracy(labels=labels,
@@ -129,9 +73,7 @@ def main(argv):
     args = parser.parse_args(argv[1:])
 
     # Fetch the data
-    (train_x, train_y), (test_x, test_y) = load_data()
-    train_x = dict(train_x)
-    test_x = dict(test_x)
+    (train_x, train_y), (test_x, test_y) = iris_data.load_data()
 
     # Feature columns describe how to use the input.
     my_feature_columns = []
@@ -151,12 +93,12 @@ def main(argv):
 
     # Train the Model.
     classifier.train(
-        input_fn=lambda:train_input_fn(train_x, train_y, args.batch_size),
+        input_fn=lambda:iris_data.train_input_fn(train_x, train_y, args.batch_size),
         steps=args.train_steps)
 
     # Evaluate the model.
     eval_result = classifier.evaluate(
-        input_fn=lambda:eval_input_fn(test_x, test_y, args.batch_size))
+        input_fn=lambda:iris_data.eval_input_fn(test_x, test_y, args.batch_size))
 
     print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**eval_result))
 
@@ -170,14 +112,18 @@ def main(argv):
     }
 
     predictions = classifier.predict(
-        input_fn=lambda:eval_input_fn(predict_x, batch_size=args.batch_size))
+        input_fn=lambda:iris_data.eval_input_fn(predict_x,
+                                                labels=None,
+                                                batch_size=args.batch_size))
 
     for pred_dict, expec in zip(predictions, expected):
         template = ('\nPrediction is "{}" ({:.1f}%), expected "{}"')
 
         class_id = pred_dict['class_ids'][0]
         probability = pred_dict['probabilities'][class_id]
-        print(template.format(SPECIES[class_id], 100 * probability, expec))
+
+        print(template.format(iris_data.SPECIES[class_id],
+                              100 * probability, expec))
 
 
 if __name__ == '__main__':

samples/core/get_started/estimator_test.py

@@ -23,6 +23,7 @@ import pandas as pd
 
 from six.moves import StringIO
 
+import iris_data
 import custom_estimator
 import premade_estimator
 
@@ -35,7 +36,7 @@ FOUR_LINES = "\n".join([
 def four_lines_data():
   text = StringIO(FOUR_LINES)
 
-  df = pd.read_csv(text, names=premade_estimator.CSV_COLUMN_NAMES)
+  df = pd.read_csv(text, names=iris_data.CSV_COLUMN_NAMES)
 
   xy = (df, df.pop("Species"))
   return xy, xy

samples/core/get_started/iris_data.py

+import pandas as pd
+import tensorflow as tf
+
+TRAIN_URL = "http://download.tensorflow.org/data/iris_training.csv"
+TEST_URL = "http://download.tensorflow.org/data/iris_test.csv"
+
+CSV_COLUMN_NAMES = ['SepalLength', 'SepalWidth',
+                    'PetalLength', 'PetalWidth', 'Species']
+SPECIES = ['Sentosa', 'Versicolor', 'Virginica']
+
+def maybe_download():
+    train_path = tf.keras.utils.get_file(TRAIN_URL.split('/')[-1], TRAIN_URL)
+    test_path = tf.keras.utils.get_file(TEST_URL.split('/')[-1], TEST_URL)
+
+    return train_path, test_path
+
+def load_data(y_name='Species'):
+    """Returns the iris dataset as (train_x, train_y), (test_x, test_y)."""
+    train_path, test_path = maybe_download()
+
+    train = pd.read_csv(train_path, names=CSV_COLUMN_NAMES, header=0)
+    train_x, train_y = train, train.pop(y_name)
+
+    test = pd.read_csv(test_path, names=CSV_COLUMN_NAMES, header=0)
+    test_x, test_y = test, test.pop(y_name)
+
+    return (train_x, train_y), (test_x, test_y)
+
+
+def train_input_fn(features, labels, batch_size):
+    """An input function for training"""
+    # Convert the inputs to a Dataset.
+    dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels))
+
+    # Shuffle, repeat, and batch the examples.
+    dataset = dataset.shuffle(1000).repeat().batch(batch_size)
+
+    # Return the read end of the pipeline.
+    return dataset.make_one_shot_iterator().get_next()
+
+
+def eval_input_fn(features, labels, batch_size):
+    """An input function for evaluation or prediction"""
+    features=dict(features)
+    if labels is None:
+        # No labels, use only features.
+        inputs = features
+    else:
+        inputs = (features, labels)
+
+    # Convert the inputs to a Dataset.
+    dataset = tf.data.Dataset.from_tensor_slices(inputs)
+
+    # Batch the examples
+    assert batch_size is not None, "batch_size must not be None"
+    dataset = dataset.batch(batch_size)
+
+    # Return the read end of the pipeline.
+    return dataset.make_one_shot_iterator().get_next()
+
+
+# The remainder of this file contains a simple example of a csv parser,
+#     implemented using a the `Dataset` class.
+
+# `tf.parse_csv` sets the types of the outputs to match the examples given in
+#     the `record_defaults` argument.
+CSV_TYPES = [[0.0], [0.0], [0.0], [0.0], [0]]
+
+def _parse_line(line):
+    # Decode the line into its fields
+    fields = tf.decode_csv(line, record_defaults=CSV_TYPES)
+
+    # Pack the result into a dictionary
+    features = dict(zip(CSV_COLUMN_NAMES, fields))
+
+    # Separate the label from the features
+    label = features.pop('Species')
+
+    return features, label
+
+
+def csv_input_fn(csv_path, batch_size):
+    # Create a dataset containing the text lines.
+    dataset = tf.data.TextLineDataset(csv_path).skip(1)
+
+    # Parse each line.
+    dataset = dataset.map(_parse_line)
+
+    # Shuffle, repeat, and batch the examples.
+    dataset = dataset.shuffle(1000).repeat().batch(batch_size)
+
+    # Return the read end of the pipeline.
+    return dataset.make_one_shot_iterator().get_next()
\ No newline at end of file

samples/core/get_started/premade_estimator.py

@@ -17,73 +17,21 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import pandas as pd
 import tensorflow as tf
 
+import iris_data
+
+
 parser = argparse.ArgumentParser()
 parser.add_argument('--batch_size', default=100, type=int, help='batch size')
 parser.add_argument('--train_steps', default=1000, type=int,
                     help='number of training steps')
 
-TRAIN_URL = "http://download.tensorflow.org/data/iris_training.csv"
-TEST_URL = "http://download.tensorflow.org/data/iris_test.csv"
-
-CSV_COLUMN_NAMES = ['SepalLength', 'SepalWidth',
-                    'PetalLength', 'PetalWidth', 'Species']
-SPECIES = ['Sentosa', 'Versicolor', 'Virginica']
-
-
-def load_data(y_name='Species'):
-    """Returns the iris dataset as (train_x, train_y), (test_x, test_y)."""
-    train_path = tf.keras.utils.get_file(TRAIN_URL.split('/')[-1], TRAIN_URL)
-    train = pd.read_csv(train_path, names=CSV_COLUMN_NAMES, header=0)
-    train_x, train_y = train, train.pop(y_name)
-
-    test_path = tf.keras.utils.get_file(TEST_URL.split('/')[-1], TEST_URL)
-    test = pd.read_csv(test_path, names=CSV_COLUMN_NAMES, header=0)
-    test_x, test_y = test, test.pop(y_name)
-
-    return (train_x, train_y), (test_x, test_y)
-
-
-def train_input_fn(features, labels, batch_size):
-    """An input function for training"""
-    # Convert the inputs to a Dataset.
-    dataset = tf.data.Dataset.from_tensor_slices((features, labels))
-
-    # Shuffle, repeat, and batch the examples.
-    dataset = dataset.shuffle(1000).repeat().batch(batch_size)
-
-    # Return the read end of the pipeline.
-    return dataset.make_one_shot_iterator().get_next()
-
-
-def eval_input_fn(features, labels=None, batch_size=None):
-    """An input function for evaluation or prediction"""
-    if labels is None:
-        # No labels, use only features.
-        inputs = features
-    else:
-        inputs = (features, labels)
-
-    # Convert the inputs to a Dataset.
-    dataset = tf.data.Dataset.from_tensor_slices(inputs)
-
-    # Batch the examples
-    assert batch_size is not None, "batch_size must not be None"
-    dataset = dataset.batch(batch_size)
-
-    # Return the read end of the pipeline.
-    return dataset.make_one_shot_iterator().get_next()
-
-
 def main(argv):
     args = parser.parse_args(argv[1:])
 
     # Fetch the data
-    (train_x, train_y), (test_x, test_y) = load_data()
-    train_x = dict(train_x)
-    test_x = dict(test_x)
+    (train_x, train_y), (test_x, test_y) = iris_data.load_data()
 
     # Feature columns describe how to use the input.
     my_feature_columns = []
@@ -100,12 +48,14 @@ def main(argv):
 
     # Train the Model.
     classifier.train(
-        input_fn=lambda:train_input_fn(train_x, train_y, args.batch_size),
+        input_fn=lambda:iris_data.train_input_fn(train_x, train_y,
+                                                 args.batch_size),
         steps=args.train_steps)
 
     # Evaluate the model.
     eval_result = classifier.evaluate(
-        input_fn=lambda:eval_input_fn(test_x, test_y, args.batch_size))
+        input_fn=lambda:iris_data.eval_input_fn(test_x, test_y,
+                                                args.batch_size))
 
     print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**eval_result))
 
@@ -119,14 +69,18 @@ def main(argv):
     }
 
     predictions = classifier.predict(
-        input_fn=lambda:eval_input_fn(predict_x, batch_size=args.batch_size))
+        input_fn=lambda:iris_data.eval_input_fn(predict_x,
+                                                labels=None,
+                                                batch_size=args.batch_size))
 
     for pred_dict, expec in zip(predictions, expected):
         template = ('\nPrediction is "{}" ({:.1f}%), expected "{}"')
 
         class_id = pred_dict['class_ids'][0]
         probability = pred_dict['probabilities'][class_id]
-        print(template.format(SPECIES[class_id], 100 * probability, expec))
+
+        print(template.format(iris_data.SPECIES[class_id],
+                              100 * probability, expec))
 
 
 if __name__ == '__main__':

samples/languages/java/README.md

+# TensorFlow for Java: Examples
+
+These examples include using pre-trained models for [image
+classification](label_image) and [object detection](object_detection),
+and driving the [training](training) of a pre-defined model - all using the
+TensorFlow Java API.
+
+The TensorFlow Java API does not have feature parity with the Python API.
+The Java API is most suitable for inference using pre-trained models
+and for training pre-defined models from a single Java process.
+
+Python will be the most convenient language for defining the
+numerical computation of a model.
+
+- [Slides](https://docs.google.com/presentation/d/e/2PACX-1vQ6DzxNTBrJo7K5P8t5_rBRGnyJoPUPBVOJR4ooHCwi4TlBFnIriFmI719rDNpcQzojqsV58aUqmBBx/pub?start=false&loop=false&delayms=3000) from January 2018.
+- See README.md in each subdirectory for details.

samples/languages/java/docker/Dockerfile

+FROM tensorflow/tensorflow:1.4.0
+WORKDIR /
+RUN apt-get update
+RUN apt-get -y install maven openjdk-8-jdk
+RUN mvn dependency:get -Dartifact=org.tensorflow:tensorflow:1.4.0
+RUN mvn dependency:get -Dartifact=org.tensorflow:proto:1.4.0
+CMD ["/bin/bash", "-l"]

samples/languages/java/docker/README.md

+Dockerfile for building an image suitable for running the Java examples.
+
+Typical usage:
+
+```
+docker build -t java-tensorflow .
+docker run -it --rm -v ${PWD}/..:/examples -w /examples java-tensorflow
+```
+
+That second command will pop you into a shell which has all
+the dependencies required to execute the scripts and Java
+examples.
+
+The script `sanity_test.sh` builds this container and runs a compilation
+check on all the maven projects.

samples/languages/java/docker/sanity_test.sh

+#!/bin/bash
+#
+# Silly sanity test
+DIR="$(cd "$(dirname "$0")" && pwd -P)"
+
+docker build -t java-tensorflow .
+docker run -it --rm -v ${PWD}/..:/examples java-tensorflow bash /examples/docker/test_inside_container.sh

samples/languages/java/docker/test_inside_container.sh

+#!/bin/bash
+
+set -ex
+
+cd /examples/label_image
+mvn compile
+
+cd /examples/object_detection
+mvn compile
+
+cd /examples/training
+mvn compile

samples/languages/java/label_image/.gitignore

+images
+src/main/resources
+target

samples/languages/java/label_image/README.md

+# Image Classification Example
+
+1. Download the model:
+   - If you have [TensorFlow 1.4+ for Python installed](https://www.tensorflow.org/install/),
+     run `python ./download.py`
+   - If not, but you have [docker](https://www.docker.com/get-docker) installed,
+     run `download.sh`.
+
+2. Compile [`LabelImage.java`](src/main/java/LabelImage.java):
+
+   ```
+   mvn compile
+   ```
+
+3. Download some sample images:
+   If you already have some images, great. Otherwise `download_sample_images.sh`
+   gets a few.
+
+3. Classify!
+
+   ```
+   mvn -q exec:java -Dexec.args="<path to image file>"
+   ```

samples/languages/java/label_image/download.py

+"""Create an image classification graph.
+
+Script to download a pre-trained image classifier and tweak it so that
+the model accepts raw bytes of an encoded image.
+
+Doing so involves some model-specific normalization of an image.
+Ideally, this would have been part of the image classifier model,
+but the particular model being used didn't include this normalization,
+so this script does the necessary tweaking.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from six.moves import urllib
+import os
+import zipfile
+import tensorflow as tf
+
+URL = 'https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip'
+LABELS_FILE = 'imagenet_comp_graph_label_strings.txt'
+GRAPH_FILE = 'tensorflow_inception_graph.pb'
+
+GRAPH_INPUT_TENSOR = 'input:0'
+GRAPH_PROBABILITIES_TENSOR = 'output:0'
+
+IMAGE_HEIGHT = 224
+IMAGE_WIDTH = 224
+MEAN = 117
+SCALE = 1
+
+LOCAL_DIR = 'src/main/resources'
+
+
+def download():
+  print('Downloading %s' % URL)
+  zip_filename, _ = urllib.request.urlretrieve(URL)
+  with zipfile.ZipFile(zip_filename) as zip:
+    zip.extract(LABELS_FILE)
+    zip.extract(GRAPH_FILE)
+    os.rename(LABELS_FILE, os.path.join(LOCAL_DIR, 'labels.txt'))
+    os.rename(GRAPH_FILE, os.path.join(LOCAL_DIR, 'graph.pb'))
+
+
+def create_graph_to_decode_and_normalize_image():
+  """See file docstring.
+
+  Returns:
+    input: The placeholder to feed the raw bytes of an encoded image.
+    y: A Tensor (the decoded, normalized image) to be fed to the graph.
+  """
+  image = tf.placeholder(tf.string, shape=(), name='encoded_image_bytes')
+  with tf.name_scope("preprocess"):
+    y = tf.image.decode_image(image, channels=3)
+    y = tf.cast(y, tf.float32)
+    y = tf.expand_dims(y, axis=0)
+    y = tf.image.resize_bilinear(y, (IMAGE_HEIGHT, IMAGE_WIDTH))
+    y = (y - MEAN) / SCALE
+  return (image, y)
+
+
+def patch_graph():
+  """Create graph.pb that applies the model in URL to raw image bytes."""
+  with tf.Graph().as_default() as g:
+    input_image, image_normalized = create_graph_to_decode_and_normalize_image()
+    original_graph_def = tf.GraphDef()
+    with open(os.path.join(LOCAL_DIR, 'graph.pb')) as f:
+      original_graph_def.ParseFromString(f.read())
+    softmax = tf.import_graph_def(
+        original_graph_def,
+        name='inception',
+        input_map={GRAPH_INPUT_TENSOR: image_normalized},
+        return_elements=[GRAPH_PROBABILITIES_TENSOR])
+    # We're constructing a graph that accepts a single image (as opposed to a
+    # batch of images), so might as well make the output be a vector of
+    # probabilities, instead of a batch of vectors with batch size 1.
+    output_probabilities = tf.squeeze(softmax, name='probabilities')
+    # Overwrite the graph.
+    with open(os.path.join(LOCAL_DIR, 'graph.pb'), 'w') as f:
+      f.write(g.as_graph_def().SerializeToString())
+    print('------------------------------------------------------------')
+    print('MODEL GRAPH  : graph.pb')
+    print('LABELS       : labels.txt')
+    print('INPUT TENSOR : %s' % input_image.op.name)
+    print('OUTPUT TENSOR: %s' % output_probabilities.op.name)
+
+
+if __name__ == '__main__':
+  if not os.path.exists(LOCAL_DIR):
+    os.makedirs(LOCAL_DIR)
+  download()
+  patch_graph()

samples/languages/java/label_image/download.sh

+#!/bin/bash
+
+DIR="$(cd "$(dirname "$0")" && pwd -P)"
+docker run -it -v ${DIR}:/x -w /x --rm tensorflow/tensorflow:1.4.0 python download.py

samples/languages/java/label_image/download_sample_images.sh

+#!/bin/bash
+DIR=$(dirname $0)
+mkdir -p ${DIR}/images
+cd ${DIR}/images
+
+# Some random images
+curl -o "porcupine.jpg" -L "https://cdn.pixabay.com/photo/2014/11/06/12/46/porcupines-519145_960_720.jpg"
+curl -o "whale.jpg" -L "https://static.pexels.com/photos/417196/pexels-photo-417196.jpeg"
+curl -o "terrier1u.jpg" -L "https://upload.wikimedia.org/wikipedia/commons/3/34/Australian_Terrier_Melly_%282%29.JPG"
+curl -o "terrier2.jpg" -L "https://cdn.pixabay.com/photo/2014/05/13/07/44/yorkshire-terrier-343198_960_720.jpg"