Merge remote-tracking branch 'tensorflow/master'

research/textsum/README.md

@@ -117,7 +117,7 @@ $ bazel-bin/textsum/seq2seq_attention \
     --log_root=textsum/log_root \
     --eval_dir=textsum/log_root/eval
 
-# Run the decode. Run it when the most is mostly converged.
+# Run the decode. Run it when the model is mostly converged.
 $ bazel-bin/textsum/seq2seq_attention \
     --mode=decode \
     --article_key=article \

research/transformer/spatial_transformer.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
+
+from six.moves import xrange
 import tensorflow as tf
 
 

samples/core/get_started/custom_estimator.py

@@ -17,67 +17,15 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import pandas as pd
 import tensorflow as tf
 
+import iris_data
+
 parser = argparse.ArgumentParser()
 parser.add_argument('--batch_size', default=100, type=int, help='batch size')
 parser.add_argument('--train_steps', default=1000, type=int,
                     help='number of training steps')
 
-TRAIN_URL = "http://download.tensorflow.org/data/iris_training.csv"
-TEST_URL = "http://download.tensorflow.org/data/iris_test.csv"
-
-CSV_COLUMN_NAMES = ['SepalLength', 'SepalWidth',
-                    'PetalLength', 'PetalWidth', 'Species']
-SPECIES = ['Sentosa', 'Versicolor', 'Virginica']
-
-
-def load_data(y_name='Species'):
-    """Returns the iris dataset as (train_x, train_y), (test_x, test_y)."""
-    train_path = tf.keras.utils.get_file(TRAIN_URL.split('/')[-1], TRAIN_URL)
-    train = pd.read_csv(train_path, names=CSV_COLUMN_NAMES, header=0)
-    train_x, train_y = train, train.pop(y_name)
-
-    test_path = tf.keras.utils.get_file(TEST_URL.split('/')[-1], TEST_URL)
-    test = pd.read_csv(test_path, names=CSV_COLUMN_NAMES, header=0)
-    test_x, test_y = test, test.pop(y_name)
-
-    return (train_x, train_y), (test_x, test_y)
-
-
-
-def train_input_fn(features, labels, batch_size):
-    """An input function for training"""
-    # Convert the inputs to a Dataset.
-    dataset = tf.data.Dataset.from_tensor_slices((features, labels))
-
-    # Shuffle, repeat, and batch the examples.
-    dataset = dataset.shuffle(1000).repeat().batch(batch_size)
-
-    # Return the read end of the pipeline.
-    return dataset.make_one_shot_iterator().get_next()
-
-
-def eval_input_fn(features, labels=None, batch_size=None):
-    """An input function for evaluation or prediction"""
-    if labels is None:
-        # No labels, use only features.
-        inputs = features
-    else:
-        inputs = (features, labels)
-
-    # Convert the inputs to a Dataset.
-    dataset = tf.data.Dataset.from_tensor_slices(inputs)
-
-    # Batch the examples
-    assert batch_size is not None, "batch_size must not be None"
-    dataset = dataset.batch(batch_size)
-
-    # Return the read end of the pipeline.
-    return dataset.make_one_shot_iterator().get_next()
-
-
 def my_model(features, labels, mode, params):
     """DNN with three hidden layers, and dropout of 0.1 probability."""
     # Create three fully connected layers each layer having a dropout
@@ -99,12 +47,8 @@ def my_model(features, labels, mode, params):
         }
         return tf.estimator.EstimatorSpec(mode, predictions=predictions)
 
-    # Convert the labels to a one-hot tensor of shape (length of features, 3)
-    # and with a on-value of 1 for each one-hot vector of length 3.
-    onehot_labels = tf.one_hot(labels, 3, 1, 0)
     # Compute loss.
-    loss = tf.losses.softmax_cross_entropy(
-        onehot_labels=onehot_labels, logits=logits)
+    loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
 
     # Compute evaluation metrics.
     accuracy = tf.metrics.accuracy(labels=labels,
@@ -129,9 +73,7 @@ def main(argv):
     args = parser.parse_args(argv[1:])
 
     # Fetch the data
-    (train_x, train_y), (test_x, test_y) = load_data()
-    train_x = dict(train_x)
-    test_x = dict(test_x)
+    (train_x, train_y), (test_x, test_y) = iris_data.load_data()
 
     # Feature columns describe how to use the input.
     my_feature_columns = []
@@ -151,12 +93,12 @@ def main(argv):
 
     # Train the Model.
     classifier.train(
-        input_fn=lambda:train_input_fn(train_x, train_y, args.batch_size),
+        input_fn=lambda:iris_data.train_input_fn(train_x, train_y, args.batch_size),
         steps=args.train_steps)
 
     # Evaluate the model.
     eval_result = classifier.evaluate(
-        input_fn=lambda:eval_input_fn(test_x, test_y, args.batch_size))
+        input_fn=lambda:iris_data.eval_input_fn(test_x, test_y, args.batch_size))
 
     print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**eval_result))
 
@@ -170,14 +112,18 @@ def main(argv):
     }
 
     predictions = classifier.predict(
-        input_fn=lambda:eval_input_fn(predict_x, batch_size=args.batch_size))
+        input_fn=lambda:iris_data.eval_input_fn(predict_x,
+                                                labels=None,
+                                                batch_size=args.batch_size))
 
     for pred_dict, expec in zip(predictions, expected):
         template = ('\nPrediction is "{}" ({:.1f}%), expected "{}"')
 
         class_id = pred_dict['class_ids'][0]
         probability = pred_dict['probabilities'][class_id]
-        print(template.format(SPECIES[class_id], 100 * probability, expec))
+
+        print(template.format(iris_data.SPECIES[class_id],
+                              100 * probability, expec))
 
 
 if __name__ == '__main__':

samples/core/get_started/estimator_test.py

@@ -23,6 +23,7 @@ import pandas as pd
 
 from six.moves import StringIO
 
+import iris_data
 import custom_estimator
 import premade_estimator
 
@@ -35,7 +36,7 @@ FOUR_LINES = "\n".join([
 def four_lines_data():
   text = StringIO(FOUR_LINES)
 
-  df = pd.read_csv(text, names=premade_estimator.CSV_COLUMN_NAMES)
+  df = pd.read_csv(text, names=iris_data.CSV_COLUMN_NAMES)
 
   xy = (df, df.pop("Species"))
   return xy, xy

samples/core/get_started/iris_data.py

+import pandas as pd
+import tensorflow as tf
+
+TRAIN_URL = "http://download.tensorflow.org/data/iris_training.csv"
+TEST_URL = "http://download.tensorflow.org/data/iris_test.csv"
+
+CSV_COLUMN_NAMES = ['SepalLength', 'SepalWidth',
+                    'PetalLength', 'PetalWidth', 'Species']
+SPECIES = ['Sentosa', 'Versicolor', 'Virginica']
+
+def maybe_download():
+    train_path = tf.keras.utils.get_file(TRAIN_URL.split('/')[-1], TRAIN_URL)
+    test_path = tf.keras.utils.get_file(TEST_URL.split('/')[-1], TEST_URL)
+
+    return train_path, test_path
+
+def load_data(y_name='Species'):
+    """Returns the iris dataset as (train_x, train_y), (test_x, test_y)."""
+    train_path, test_path = maybe_download()
+
+    train = pd.read_csv(train_path, names=CSV_COLUMN_NAMES, header=0)
+    train_x, train_y = train, train.pop(y_name)
+
+    test = pd.read_csv(test_path, names=CSV_COLUMN_NAMES, header=0)
+    test_x, test_y = test, test.pop(y_name)
+
+    return (train_x, train_y), (test_x, test_y)
+
+
+def train_input_fn(features, labels, batch_size):
+    """An input function for training"""
+    # Convert the inputs to a Dataset.
+    dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels))
+
+    # Shuffle, repeat, and batch the examples.
+    dataset = dataset.shuffle(1000).repeat().batch(batch_size)
+
+    # Return the read end of the pipeline.
+    return dataset.make_one_shot_iterator().get_next()
+
+
+def eval_input_fn(features, labels, batch_size):
+    """An input function for evaluation or prediction"""
+    features=dict(features)
+    if labels is None:
+        # No labels, use only features.
+        inputs = features
+    else:
+        inputs = (features, labels)
+
+    # Convert the inputs to a Dataset.
+    dataset = tf.data.Dataset.from_tensor_slices(inputs)
+
+    # Batch the examples
+    assert batch_size is not None, "batch_size must not be None"
+    dataset = dataset.batch(batch_size)
+
+    # Return the read end of the pipeline.
+    return dataset.make_one_shot_iterator().get_next()
+
+
+# The remainder of this file contains a simple example of a csv parser,
+#     implemented using a the `Dataset` class.
+
+# `tf.parse_csv` sets the types of the outputs to match the examples given in
+#     the `record_defaults` argument.
+CSV_TYPES = [[0.0], [0.0], [0.0], [0.0], [0]]
+
+def _parse_line(line):
+    # Decode the line into its fields
+    fields = tf.decode_csv(line, record_defaults=CSV_TYPES)
+
+    # Pack the result into a dictionary
+    features = dict(zip(CSV_COLUMN_NAMES, fields))
+
+    # Separate the label from the features
+    label = features.pop('Species')
+
+    return features, label
+
+
+def csv_input_fn(csv_path, batch_size):
+    # Create a dataset containing the text lines.
+    dataset = tf.data.TextLineDataset(csv_path).skip(1)
+
+    # Parse each line.
+    dataset = dataset.map(_parse_line)
+
+    # Shuffle, repeat, and batch the examples.
+    dataset = dataset.shuffle(1000).repeat().batch(batch_size)
+
+    # Return the read end of the pipeline.
+    return dataset.make_one_shot_iterator().get_next()
\ No newline at end of file

samples/core/get_started/premade_estimator.py

@@ -17,73 +17,21 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import pandas as pd
 import tensorflow as tf
 
+import iris_data
+
+
 parser = argparse.ArgumentParser()
 parser.add_argument('--batch_size', default=100, type=int, help='batch size')
 parser.add_argument('--train_steps', default=1000, type=int,
                     help='number of training steps')
 
-TRAIN_URL = "http://download.tensorflow.org/data/iris_training.csv"
-TEST_URL = "http://download.tensorflow.org/data/iris_test.csv"
-
-CSV_COLUMN_NAMES = ['SepalLength', 'SepalWidth',
-                    'PetalLength', 'PetalWidth', 'Species']
-SPECIES = ['Sentosa', 'Versicolor', 'Virginica']
-
-
-def load_data(y_name='Species'):
-    """Returns the iris dataset as (train_x, train_y), (test_x, test_y)."""
-    train_path = tf.keras.utils.get_file(TRAIN_URL.split('/')[-1], TRAIN_URL)
-    train = pd.read_csv(train_path, names=CSV_COLUMN_NAMES, header=0)
-    train_x, train_y = train, train.pop(y_name)
-
-    test_path = tf.keras.utils.get_file(TEST_URL.split('/')[-1], TEST_URL)
-    test = pd.read_csv(test_path, names=CSV_COLUMN_NAMES, header=0)
-    test_x, test_y = test, test.pop(y_name)
-
-    return (train_x, train_y), (test_x, test_y)
-
-
-def train_input_fn(features, labels, batch_size):
-    """An input function for training"""
-    # Convert the inputs to a Dataset.
-    dataset = tf.data.Dataset.from_tensor_slices((features, labels))
-
-    # Shuffle, repeat, and batch the examples.
-    dataset = dataset.shuffle(1000).repeat().batch(batch_size)
-
-    # Return the read end of the pipeline.
-    return dataset.make_one_shot_iterator().get_next()
-
-
-def eval_input_fn(features, labels=None, batch_size=None):
-    """An input function for evaluation or prediction"""
-    if labels is None:
-        # No labels, use only features.
-        inputs = features
-    else:
-        inputs = (features, labels)
-
-    # Convert the inputs to a Dataset.
-    dataset = tf.data.Dataset.from_tensor_slices(inputs)
-
-    # Batch the examples
-    assert batch_size is not None, "batch_size must not be None"
-    dataset = dataset.batch(batch_size)
-
-    # Return the read end of the pipeline.
-    return dataset.make_one_shot_iterator().get_next()
-
-
 def main(argv):
     args = parser.parse_args(argv[1:])
 
     # Fetch the data
-    (train_x, train_y), (test_x, test_y) = load_data()
-    train_x = dict(train_x)
-    test_x = dict(test_x)
+    (train_x, train_y), (test_x, test_y) = iris_data.load_data()
 
     # Feature columns describe how to use the input.
     my_feature_columns = []
@@ -100,12 +48,14 @@ def main(argv):
 
     # Train the Model.
     classifier.train(
-        input_fn=lambda:train_input_fn(train_x, train_y, args.batch_size),
+        input_fn=lambda:iris_data.train_input_fn(train_x, train_y,
+                                                 args.batch_size),
         steps=args.train_steps)
 
     # Evaluate the model.
     eval_result = classifier.evaluate(
-        input_fn=lambda:eval_input_fn(test_x, test_y, args.batch_size))
+        input_fn=lambda:iris_data.eval_input_fn(test_x, test_y,
+                                                args.batch_size))
 
     print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**eval_result))
 
@@ -119,14 +69,18 @@ def main(argv):
     }
 
     predictions = classifier.predict(
-        input_fn=lambda:eval_input_fn(predict_x, batch_size=args.batch_size))
+        input_fn=lambda:iris_data.eval_input_fn(predict_x,
+                                                labels=None,
+                                                batch_size=args.batch_size))
 
     for pred_dict, expec in zip(predictions, expected):
         template = ('\nPrediction is "{}" ({:.1f}%), expected "{}"')
 
         class_id = pred_dict['class_ids'][0]
         probability = pred_dict['probabilities'][class_id]
-        print(template.format(SPECIES[class_id], 100 * probability, expec))
+
+        print(template.format(iris_data.SPECIES[class_id],
+                              100 * probability, expec))
 
 
 if __name__ == '__main__':

samples/outreach/blogs/blog_custom_estimators.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# Distributed_TensorFlow under the License is Distributed_TensorFlow on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# This is the complete code for the following blogpost:
+# https://developers.googleblog.com/2017/12/creating-custom-estimators-in-tensorflow.html
+
+import tensorflow as tf
+import os
+import sys
+
+import six.moves.urllib.request as request
+
+tf.logging.set_verbosity(tf.logging.INFO)
+
+# Check that we have correct TensorFlow version installed
+tf_version = tf.__version__
+tf.logging.info("TensorFlow version: {}".format(tf_version))
+assert "1.4" <= tf_version, "TensorFlow r1.4 or later is needed"
+
+# Windows users: You only need to change PATH, rest is platform independent
+PATH = "/tmp/tf_custom_estimators"
+
+# Fetch and store Training and Test dataset files
+PATH_DATASET = PATH + os.sep + "dataset"
+FILE_TRAIN = PATH_DATASET + os.sep + "iris_training.csv"
+FILE_TEST = PATH_DATASET + os.sep + "iris_test.csv"
+URL_TRAIN = "http://download.tensorflow.org/data/iris_training.csv"
+URL_TEST = "http://download.tensorflow.org/data/iris_test.csv"
+
+def downloadDataset(url, file):
+    if not os.path.exists(PATH_DATASET):
+        os.makedirs(PATH_DATASET)
+    if not os.path.exists(file):
+        data = request.urlopen(url).read()
+        with open(file, "wb") as f:
+            f.write(data)
+            f.close()
+downloadDataset(URL_TRAIN, FILE_TRAIN)
+downloadDataset(URL_TEST, FILE_TEST)
+
+# The CSV features in our training & test data
+feature_names = [
+    'SepalLength',
+    'SepalWidth',
+    'PetalLength',
+    'PetalWidth']
+
+# Create an input function reading a file using the Dataset API
+# Then provide the results to the Estimator API
+def my_input_fn(file_path, repeat_count=1, shuffle_count=1):
+    def decode_csv(line):
+        parsed_line = tf.decode_csv(line, [[0.], [0.], [0.], [0.], [0]])
+        label = parsed_line[-1]  # Last element is the label
+        del parsed_line[-1]  # Delete last element
+        features = parsed_line  # Everything but last elements are the features
+        d = dict(zip(feature_names, features)), label
+        return d
+
+    dataset = (tf.data.TextLineDataset(file_path)  # Read text file
+        .skip(1)  # Skip header row
+        .map(decode_csv, num_parallel_calls=4)  # Decode each line
+        .cache() # Warning: Caches entire dataset, can cause out of memory
+        .shuffle(shuffle_count)  # Randomize elems (1 == no operation)
+        .repeat(repeat_count)    # Repeats dataset this # times
+        .batch(32)
+        .prefetch(1)  # Make sure you always have 1 batch ready to serve
+    )
+    iterator = dataset.make_one_shot_iterator()
+    batch_features, batch_labels = iterator.get_next()
+    return batch_features, batch_labels
+
+def my_model_fn(
+    features, # This is batch_features from input_fn
+    labels,   # This is batch_labels from input_fn
+    mode):    # And instance of tf.estimator.ModeKeys, see below
+
+    if mode == tf.estimator.ModeKeys.PREDICT:
+        tf.logging.info("my_model_fn: PREDICT, {}".format(mode))
+    elif mode == tf.estimator.ModeKeys.EVAL:
+        tf.logging.info("my_model_fn: EVAL, {}".format(mode))
+    elif mode == tf.estimator.ModeKeys.TRAIN:
+        tf.logging.info("my_model_fn: TRAIN, {}".format(mode))
+
+    # All our inputs are feature columns of type numeric_column
+    feature_columns = [
+        tf.feature_column.numeric_column(feature_names[0]),
+        tf.feature_column.numeric_column(feature_names[1]),
+        tf.feature_column.numeric_column(feature_names[2]),
+        tf.feature_column.numeric_column(feature_names[3])
+    ]
+
+    # Create the layer of input
+    input_layer = tf.feature_column.input_layer(features, feature_columns)
+
+    # Definition of hidden layer: h1
+    # We implement it as a fully-connected layer (tf.layers.dense)
+    # Has 10 neurons, and uses ReLU as the activation function
+    # Takes input_layer as input
+    h1 = tf.layers.Dense(10, activation=tf.nn.relu)(input_layer)
+
+    # Definition of hidden layer: h2 (this is the logits layer)
+    # Similar to h1, but takes h1 as input
+    h2 = tf.layers.Dense(10, activation=tf.nn.relu)(h1)
+
+    # Output 'logits' layer is three number = probability distribution
+    # between Iris Setosa, Versicolor, and Viginica
+    logits = tf.layers.Dense(3)(h2)
+
+    # class_ids will be the model prediction for the class (Iris flower type)
+    # The output node with the highest value is our prediction
+    predictions = { 'class_ids': tf.argmax(input=logits, axis=1) }
+
+    # 1. Prediction mode
+    # Return our prediction
+    if mode == tf.estimator.ModeKeys.PREDICT:
+        return tf.estimator.EstimatorSpec(mode, predictions=predictions)
+
+    # Evaluation and Training mode
+
+    # Calculate the loss
+    loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
+
+    # Calculate the accuracy between the true labels, and our predictions
+    accuracy = tf.metrics.accuracy(labels, predictions['class_ids'])
+
+    # 2. Evaluation mode
+    # Return our loss (which is used to evaluate our model)
+    # Set the TensorBoard scalar my_accurace to the accuracy
+    # Obs: This function only sets value during mode == ModeKeys.EVAL
+    # To set values during training, see tf.summary.scalar
+    if mode == tf.estimator.ModeKeys.EVAL:
+        return tf.estimator.EstimatorSpec(
+            mode,
+            loss=loss,
+            eval_metric_ops={'my_accuracy': accuracy})
+
+    # If mode is not PREDICT nor EVAL, then we must be in TRAIN
+    assert mode == tf.estimator.ModeKeys.TRAIN, "TRAIN is only ModeKey left"
+
+    # 3. Training mode
+
+    # Default optimizer for DNNClassifier: Adagrad with learning rate=0.05
+    # Our objective (train_op) is to minimize loss
+    # Provide global step counter (used to count gradient updates)
+    optimizer = tf.train.AdagradOptimizer(0.05)
+    train_op = optimizer.minimize(
+        loss,
+        global_step=tf.train.get_global_step())
+
+    # Set the TensorBoard scalar my_accuracy to the accuracy
+    # Obs: This function only sets the value during mode == ModeKeys.TRAIN
+    # To set values during evaluation, see eval_metrics_ops
+    tf.summary.scalar('my_accuracy', accuracy[1])
+
+    # Return training operations: loss and train_op
+    return tf.estimator.EstimatorSpec(
+        mode,
+        loss=loss,
+        train_op=train_op)
+
+# Create a custom estimator using my_model_fn to define the model
+tf.logging.info("Before classifier construction")
+classifier = tf.estimator.Estimator(
+    model_fn=my_model_fn,
+    model_dir=PATH)  # Path to where checkpoints etc are stored
+tf.logging.info("...done constructing classifier")
+
+# 500 epochs = 500 * 120 records [60000] = (500 * 120) / 32 batches = 1875 batches
+# 4 epochs = 4 * 30 records = (4 * 30) / 32 batches = 3.75 batches
+
+# Train our model, use the previously function my_input_fn
+# Input to training is a file with training example
+# Stop training after 8 iterations of train data (epochs)
+tf.logging.info("Before classifier.train")
+classifier.train(
+    input_fn=lambda: my_input_fn(FILE_TRAIN, 500, 256))
+tf.logging.info("...done classifier.train")
+
+# Evaluate our model using the examples contained in FILE_TEST
+# Return value will contain evaluation_metrics such as: loss & average_loss
+tf.logging.info("Before classifier.evaluate")
+evaluate_result = classifier.evaluate(
+    input_fn=lambda: my_input_fn(FILE_TEST, 4))
+tf.logging.info("...done classifier.evaluate")
+tf.logging.info("Evaluation results")
+for key in evaluate_result:
+    tf.logging.info("   {}, was: {}".format(key, evaluate_result[key]))
+
+# Predict the type of some Iris flowers.
+# Let's predict the examples in FILE_TEST, repeat only once.
+predict_results = classifier.predict(
+    input_fn=lambda: my_input_fn(FILE_TEST, 1))
+tf.logging.info("Prediction on test file")
+for prediction in predict_results:
+    # Will print the predicted class, i.e: 0, 1, or 2 if the prediction
+    # is Iris Setosa, Vericolor, Virginica, respectively.
+    tf.logging.info("...{}".format(prediction["class_ids"]))
+
+# Let create a dataset for prediction
+# We've taken the first 3 examples in FILE_TEST
+prediction_input = [[5.9, 3.0, 4.2, 1.5],  # -> 1, Iris Versicolor
+                    [6.9, 3.1, 5.4, 2.1],  # -> 2, Iris Virginica
+                    [5.1, 3.3, 1.7, 0.5]]  # -> 0, Iris Setosa
+
+def new_input_fn():
+    def decode(x):
+        x = tf.split(x, 4)  # Need to split into our 4 features
+        return dict(zip(feature_names, x))  # To build a dict of them
+
+    dataset = tf.data.Dataset.from_tensor_slices(prediction_input)
+    dataset = dataset.map(decode)
+    iterator = dataset.make_one_shot_iterator()
+    next_feature_batch = iterator.get_next()
+    return next_feature_batch, None  # In prediction, we have no labels
+
+# Predict all our prediction_input
+predict_results = classifier.predict(input_fn=new_input_fn)
+
+# Print results
+tf.logging.info("Predictions on memory")
+for idx, prediction in enumerate(predict_results):
+    type = prediction["class_ids"]  # Get the predicted class (index)
+    if type == 0:
+        tf.logging.info("...I think: {}, is Iris Setosa".format(prediction_input[idx]))
+    elif type == 1:
+        tf.logging.info("...I think: {}, is Iris Versicolor".format(prediction_input[idx]))
+    else:
+        tf.logging.info("...I think: {}, is Iris Virginica".format(prediction_input[idx]))

tutorials/image/cifar10/README.md

+**NOTE: For users interested in multi-GPU, we recommend looking at the newer [cifar10_estimator](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator) example instead.**
+
+---
+
 CIFAR-10 is a common benchmark in machine learning for image recognition.
 
 http://www.cs.toronto.edu/~kriz/cifar.html
@@ -7,4 +11,3 @@ Code in this directory demonstrates how to use TensorFlow to train and evaluate
 Detailed instructions on how to get started available at:
 
 http://tensorflow.org/tutorials/deep_cnn/
-

tutorials/image/cifar10/cifar10.py

@@ -35,7 +35,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import argparse
 import os
 import re
 import sys
@@ -46,19 +45,15 @@ import tensorflow as tf
 
 import cifar10_input
 
-parser = argparse.ArgumentParser()
+FLAGS = tf.app.flags.FLAGS
 
 # Basic model parameters.
-parser.add_argument('--batch_size', type=int, default=128,
-                    help='Number of images to process in a batch.')
-
-parser.add_argument('--data_dir', type=str, default='/tmp/cifar10_data',
-                    help='Path to the CIFAR-10 data directory.')
-
-parser.add_argument('--use_fp16', type=bool, default=False,
-                    help='Train the model using fp16.')
-
-FLAGS = parser.parse_args()
+tf.app.flags.DEFINE_integer('batch_size', 128,
+                            """Number of images to process in a batch.""")
+tf.app.flags.DEFINE_string('data_dir', '/tmp/cifar10_data',
+                           """Path to the CIFAR-10 data directory.""")
+tf.app.flags.DEFINE_boolean('use_fp16', False,
+                            """Train the model using fp16.""")
 
 # Global constants describing the CIFAR-10 data set.
 IMAGE_SIZE = cifar10_input.IMAGE_SIZE
@@ -78,7 +73,7 @@ INITIAL_LEARNING_RATE = 0.1       # Initial learning rate.
 # names of the summaries when visualizing a model.
 TOWER_NAME = 'tower'
 
-DATA_URL = 'http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz'
+DATA_URL = 'https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz'
 
 
 def _activation_summary(x):

tutorials/image/cifar10/cifar10_eval.py

@@ -43,25 +43,20 @@ import tensorflow as tf
 
 import cifar10
 
-parser = cifar10.parser
-
-parser.add_argument('--eval_dir', type=str, default='/tmp/cifar10_eval',
-                    help='Directory where to write event logs.')
-
-parser.add_argument('--eval_data', type=str, default='test',
-                    help='Either `test` or `train_eval`.')
-
-parser.add_argument('--checkpoint_dir', type=str, default='/tmp/cifar10_train',
-                    help='Directory where to read model checkpoints.')
-
-parser.add_argument('--eval_interval_secs', type=int, default=60*5,
-                    help='How often to run the eval.')
-
-parser.add_argument('--num_examples', type=int, default=10000,
-                    help='Number of examples to run.')
-
-parser.add_argument('--run_once', type=bool, default=False,
-                    help='Whether to run eval only once.')
+FLAGS = tf.app.flags.FLAGS
+
+tf.app.flags.DEFINE_string('eval_dir', '/tmp/cifar10_eval',
+                           """Directory where to write event logs.""")
+tf.app.flags.DEFINE_string('eval_data', 'test',
+                           """Either 'test' or 'train_eval'.""")
+tf.app.flags.DEFINE_string('checkpoint_dir', '/tmp/cifar10_train',
+                           """Directory where to read model checkpoints.""")
+tf.app.flags.DEFINE_integer('eval_interval_secs', 60 * 5,
+                            """How often to run the eval.""")
+tf.app.flags.DEFINE_integer('num_examples', 10000,
+                            """Number of examples to run.""")
+tf.app.flags.DEFINE_boolean('run_once', False,
+                         """Whether to run eval only once.""")
 
 
 def eval_once(saver, summary_writer, top_k_op, summary_op):
@@ -159,5 +154,4 @@ def main(argv=None):  # pylint: disable=unused-argument
 
 
 if __name__ == '__main__':
-  FLAGS = parser.parse_args()
   tf.app.run()

tutorials/image/cifar10/cifar10_multi_gpu_train.py

@@ -49,19 +49,17 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 import tensorflow as tf
 import cifar10
 
-parser = cifar10.parser
+FLAGS = tf.app.flags.FLAGS
 
-parser.add_argument('--train_dir', type=str, default='/tmp/cifar10_train',
-                    help='Directory where to write event logs and checkpoint.')
-
-parser.add_argument('--max_steps', type=int, default=1000000,
-                    help='Number of batches to run.')
-
-parser.add_argument('--num_gpus', type=int, default=1,
-                    help='How many GPUs to use.')
-
-parser.add_argument('--log_device_placement', type=bool, default=False,
-                    help='Whether to log device placement.')
+tf.app.flags.DEFINE_string('train_dir', '/tmp/cifar10_train',
+                           """Directory where to write event logs """
+                           """and checkpoint.""")
+tf.app.flags.DEFINE_integer('max_steps', 1000000,
+                            """Number of batches to run.""")
+tf.app.flags.DEFINE_integer('num_gpus', 1,
+                            """How many GPUs to use.""")
+tf.app.flags.DEFINE_boolean('log_device_placement', False,
+                            """Whether to log device placement.""")
 
 
 def tower_loss(scope, images, labels):
@@ -276,5 +274,4 @@ def main(argv=None):  # pylint: disable=unused-argument
 
 
 if __name__ == '__main__':
-  FLAGS = parser.parse_args()
   tf.app.run()

tutorials/image/cifar10/cifar10_train.py

@@ -43,19 +43,17 @@ import tensorflow as tf
 
 import cifar10
 
-parser = cifar10.parser
+FLAGS = tf.app.flags.FLAGS
 
-parser.add_argument('--train_dir', type=str, default='/tmp/cifar10_train',
-                    help='Directory where to write event logs and checkpoint.')
-
-parser.add_argument('--max_steps', type=int, default=1000000,
-                    help='Number of batches to run.')
-
-parser.add_argument('--log_device_placement', type=bool, default=False,
-                    help='Whether to log device placement.')
-
-parser.add_argument('--log_frequency', type=int, default=10,
-                    help='How often to log results to the console.')
+tf.app.flags.DEFINE_string('train_dir', '/tmp/cifar10_train',
+                           """Directory where to write event logs """
+                           """and checkpoint.""")
+tf.app.flags.DEFINE_integer('max_steps', 1000000,
+                            """Number of batches to run.""")
+tf.app.flags.DEFINE_boolean('log_device_placement', False,
+                            """Whether to log device placement.""")
+tf.app.flags.DEFINE_integer('log_frequency', 10,
+                            """How often to log results to the console.""")
 
 
 def train():
@@ -126,5 +124,4 @@ def main(argv=None):  # pylint: disable=unused-argument
 
 
 if __name__ == '__main__':
-  FLAGS = parser.parse_args()
   tf.app.run()

tutorials/rnn/README.md

@@ -4,6 +4,7 @@ and use them are available in the tutorials.
 
 * [RNN Tutorial](http://tensorflow.org/tutorials/recurrent/)
 * [Sequence-to-Sequence Tutorial](http://tensorflow.org/tutorials/seq2seq/)
+* [RNN Tutorial for Drawing Classification](https://www.tensorflow.org/versions/master/tutorials/recurrent_quickdraw)
 
 Here is a short overview of what is in this directory.
 
@@ -11,3 +12,4 @@ File | What's in it?
 --- | ---
 `ptb/` | PTB language model, see the [RNN Tutorial](http://tensorflow.org/tutorials/recurrent/)
 `translate/` | Translation model, see the [Sequence-to-Sequence Tutorial](http://tensorflow.org/tutorials/seq2seq/)
+`quickdraw/` | Quick, Draw! model, see the [RNN Tutorial for Drawing Classification](https://www.tensorflow.org/versions/master/tutorials/recurrent_quickdraw)

tutorials/rnn/ptb/ptb_word_lm.py

@@ -213,13 +213,15 @@ class PTBModel(object):
     # Slightly better results can be obtained with forget gate biases
     # initialized to 1 but the hyperparameters of the model would need to be
     # different than reported in the paper.
-    cell = self._get_lstm_cell(config, is_training)
-    if is_training and config.keep_prob < 1:
-      cell = tf.contrib.rnn.DropoutWrapper(
-          cell, output_keep_prob=config.keep_prob)
+    def make_cell():
+      cell = self._get_lstm_cell(config, is_training)
+      if is_training and config.keep_prob < 1:
+        cell = tf.contrib.rnn.DropoutWrapper(
+            cell, output_keep_prob=config.keep_prob)
+      return cell
 
     cell = tf.contrib.rnn.MultiRNNCell(
-        [cell for _ in range(config.num_layers)], state_is_tuple=True)
+        [make_cell() for _ in range(config.num_layers)], state_is_tuple=True)
 
     self._initial_state = cell.zero_state(config.batch_size, data_type())
     state = self._initial_state
@@ -329,7 +331,7 @@ class SmallConfig(object):
   lr_decay = 0.5
   batch_size = 20
   vocab_size = 10000
-  rnn_mode = CUDNN
+  rnn_mode = BLOCK
 
 
 class MediumConfig(object):

tutorials/rnn/quickdraw/BUILD

+# Description:
+# Example classification model on Quick, Draw! dataset.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+py_binary(
+    name = "train_model",
+    srcs = [
+        "train_model.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//third_party/py/tensorflow",
+    ],
+)
+
+py_binary(
+    name = "create_dataset",
+    srcs = [
+        "create_dataset.py",
+    ],
+    deps = [
+        "//third_party/py/numpy",
+        "//third_party/py/tensorflow",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//third_party/tensorflow:__subpackages__"],
+)

tutorials/rnn/quickdraw/create_dataset.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+r"""Creates training and eval data from Quickdraw NDJSON files.
+
+This tool reads the NDJSON files from https://quickdraw.withgoogle.com/data
+and converts them into tensorflow.Example stored in TFRecord files.
+
+The tensorflow example will contain 3 features:
+ shape - contains the shape of the sequence [length, dim] where dim=3.
+ class_index - the class index of the class for the example.
+ ink - a length * dim vector of the ink.
+
+It creates disjoint training and evaluation sets.
+
+python create_dataset.py \
+  --ndjson_path ${HOME}/ndjson \
+  --output_path ${HOME}/tfrecord
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import json
+import os
+import random
+import sys
+import numpy as np
+import tensorflow as tf
+
+
+def parse_line(ndjson_line):
+  """Parse an ndjson line and return ink (as np array) and classname."""
+  sample = json.loads(ndjson_line)
+  class_name = sample["word"]
+  if not class_name:
+    print ("Empty classname")
+    return None, None
+  inkarray = sample["drawing"]
+  stroke_lengths = [len(stroke[0]) for stroke in inkarray]
+  total_points = sum(stroke_lengths)
+  np_ink = np.zeros((total_points, 3), dtype=np.float32)
+  current_t = 0
+  if not inkarray:
+    print("Empty inkarray")
+    return None, None
+  for stroke in inkarray:
+    if len(stroke[0]) != len(stroke[1]):
+      print("Inconsistent number of x and y coordinates.")
+      return None, None
+    for i in [0, 1]:
+      np_ink[current_t:(current_t + len(stroke[0])), i] = stroke[i]
+    current_t += len(stroke[0])
+    np_ink[current_t - 1, 2] = 1  # stroke_end
+  # Preprocessing.
+  # 1. Size normalization.
+  lower = np.min(np_ink[:, 0:2], axis=0)
+  upper = np.max(np_ink[:, 0:2], axis=0)
+  scale = upper - lower
+  scale[scale == 0] = 1
+  np_ink[:, 0:2] = (np_ink[:, 0:2] - lower) / scale
+  # 2. Compute deltas.
+  np_ink[1:, 0:2] -= np_ink[0:-1, 0:2]
+  np_ink = np_ink[1:, :]
+  return np_ink, class_name
+
+
+def convert_data(trainingdata_dir,
+                 observations_per_class,
+                 output_file,
+                 classnames,
+                 output_shards=10,
+                 offset=0):
+  """Convert training data from ndjson files into tf.Example in tf.Record.
+
+  Args:
+   trainingdata_dir: path to the directory containin the training data.
+     The training data is stored in that directory as ndjson files.
+   observations_per_class: the number of items to load per class.
+   output_file: path where to write the output.
+   classnames: array with classnames - is auto created if not passed in.
+   output_shards: the number of shards to write the output in.
+   offset: the number of items to skip at the beginning of each file.
+
+  Returns:
+    classnames: the class names as strings. classnames[classes[i]] is the
+      textual representation of the class of the i-th data point.
+  """
+
+  def _pick_output_shard():
+    return random.randint(0, output_shards - 1)
+
+  file_handles = []
+  # Open all input files.
+  for filename in sorted(tf.gfile.ListDirectory(trainingdata_dir)):
+    if not filename.endswith(".ndjson"):
+      print("Skipping", filename)
+      continue
+    file_handles.append(
+        tf.gfile.GFile(os.path.join(trainingdata_dir, filename), "r"))
+    if offset:  # Fast forward all files to skip the offset.
+      count = 0
+      for _ in file_handles[-1]:
+        count += 1
+        if count == offset:
+          break
+
+  writers = []
+  for i in range(FLAGS.output_shards):
+    writers.append(
+        tf.python_io.TFRecordWriter("%s-%05i-of-%05i" % (output_file, i,
+                                                         output_shards)))
+
+  reading_order = range(len(file_handles)) * observations_per_class
+  random.shuffle(reading_order)
+
+  for c in reading_order:
+    line = file_handles[c].readline()
+    ink = None
+    while ink is None:
+      ink, class_name = parse_line(line)
+      if ink is None:
+        print ("Couldn't parse ink from '" + line + "'.")
+    if class_name not in classnames:
+      classnames.append(class_name)
+    features = {}
+    features["class_index"] = tf.train.Feature(int64_list=tf.train.Int64List(
+        value=[classnames.index(class_name)]))
+    features["ink"] = tf.train.Feature(float_list=tf.train.FloatList(
+        value=ink.flatten()))
+    features["shape"] = tf.train.Feature(int64_list=tf.train.Int64List(
+        value=ink.shape))
+    f = tf.train.Features(feature=features)
+    example = tf.train.Example(features=f)
+    writers[_pick_output_shard()].write(example.SerializeToString())
+
+  # Close all files
+  for w in writers:
+    w.close()
+  for f in file_handles:
+    f.close()
+  # Write the class list.
+  with tf.gfile.GFile(output_file + ".classes", "w") as f:
+    for class_name in classnames:
+      f.write(class_name + "\n")
+  return classnames
+
+
+def main(argv):
+  del argv
+  classnames = convert_data(
+      FLAGS.ndjson_path,
+      FLAGS.train_observations_per_class,
+      os.path.join(FLAGS.output_path, "training.tfrecord"),
+      classnames=[],
+      output_shards=FLAGS.output_shards,
+      offset=0)
+  convert_data(
+      FLAGS.ndjson_path,
+      FLAGS.eval_observations_per_class,
+      os.path.join(FLAGS.output_path, "eval.tfrecord"),
+      classnames=classnames,
+      output_shards=FLAGS.output_shards,
+      offset=FLAGS.train_observations_per_class)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.register("type", "bool", lambda v: v.lower() == "true")
+  parser.add_argument(
+      "--ndjson_path",
+      type=str,
+      default="",
+      help="Directory where the ndjson files are stored.")
+  parser.add_argument(
+      "--output_path",
+      type=str,
+      default="",
+      help="Directory where to store the output TFRecord files.")
+  parser.add_argument(
+      "--train_observations_per_class",
+      type=int,
+      default=10000,
+      help="How many items per class to load for training.")
+  parser.add_argument(
+      "--eval_observations_per_class",
+      type=int,
+      default=1000,
+      help="How many items per class to load for evaluation.")
+  parser.add_argument(
+      "--output_shards",
+      type=int,
+      default=10,
+      help="Number of shards for the output.")
+
+  FLAGS, unparsed = parser.parse_known_args()
+  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)

tutorials/rnn/quickdraw/train_model.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+r"""Binary for trianing a RNN-based classifier for the Quick, Draw! data.
+
+python train_model.py \
+  --training_data train_data \
+  --eval_data eval_data \
+  --model_dir /tmp/quickdraw_model/ \
+  --cell_type cudnn_lstm
+
+When running on GPUs using --cell_type cudnn_lstm is much faster.
+
+The expected performance is ~75% in 1.5M steps with the default configuration.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import argparse
+import ast
+import functools
+import sys
+
+import tensorflow as tf
+
+
+def get_num_classes():
+  classes = []
+  with tf.gfile.GFile(FLAGS.classes_file, "r") as f:
+    classes = [x for x in f]
+  num_classes = len(classes)
+  return num_classes
+
+
+def get_input_fn(mode, tfrecord_pattern, batch_size):
+  """Creates an input_fn that stores all the data in memory.
+
+  Args:
+   mode: one of tf.contrib.learn.ModeKeys.{TRAIN, INFER, EVAL}
+   tfrecord_pattern: path to a TF record file created using create_dataset.py.
+   batch_size: the batch size to output.
+
+  Returns:
+    A valid input_fn for the model estimator.
+  """
+
+  def _parse_tfexample_fn(example_proto, mode):
+    """Parse a single record which is expected to be a tensorflow.Example."""
+    feature_to_type = {
+        "ink": tf.VarLenFeature(dtype=tf.float32),
+        "shape": tf.FixedLenFeature([2], dtype=tf.int64)
+    }
+    if mode != tf.estimator.ModeKeys.PREDICT:
+      # The labels won't be available at inference time, so don't add them
+      # to the list of feature_columns to be read.
+      feature_to_type["class_index"] = tf.FixedLenFeature([1], dtype=tf.int64)
+
+    parsed_features = tf.parse_single_example(example_proto, feature_to_type)
+    labels = None
+    if mode != tf.estimator.ModeKeys.PREDICT:
+      labels = parsed_features["class_index"]
+    parsed_features["ink"] = tf.sparse_tensor_to_dense(parsed_features["ink"])
+    return parsed_features, labels
+
+  def _input_fn():
+    """Estimator `input_fn`.
+
+    Returns:
+      A tuple of:
+      - Dictionary of string feature name to `Tensor`.
+      - `Tensor` of target labels.
+    """
+    dataset = tf.data.TFRecordDataset.list_files(tfrecord_pattern)
+    if mode == tf.estimator.ModeKeys.TRAIN:
+      dataset = dataset.shuffle(buffer_size=10)
+    dataset = dataset.repeat()
+    # Preprocesses 10 files concurrently and interleaves records from each file.
+    dataset = dataset.interleave(
+        tf.data.TFRecordDataset,
+        cycle_length=10,
+        block_length=1)
+    dataset = dataset.map(
+        functools.partial(_parse_tfexample_fn, mode=mode),
+        num_parallel_calls=10)
+    dataset = dataset.prefetch(10000)
+    if mode == tf.estimator.ModeKeys.TRAIN:
+      dataset = dataset.shuffle(buffer_size=1000000)
+    # Our inputs are variable length, so pad them.
+    dataset = dataset.padded_batch(
+        batch_size, padded_shapes=dataset.output_shapes)
+    features, labels = dataset.make_one_shot_iterator().get_next()
+    return features, labels
+
+  return _input_fn
+
+
+def model_fn(features, labels, mode, params):
+  """Model function for RNN classifier.
+
+  This function sets up a neural network which applies convolutional layers (as
+  configured with params.num_conv and params.conv_len) to the input.
+  The output of the convolutional layers is given to LSTM layers (as configured
+  with params.num_layers and params.num_nodes).
+  The final state of the all LSTM layers are concatenated and fed to a fully
+  connected layer to obtain the final classification scores.
+
+  Args:
+    features: dictionary with keys: inks, lengths.
+    labels: one hot encoded classes
+    mode: one of tf.estimator.ModeKeys.{TRAIN, INFER, EVAL}
+    params: a parameter dictionary with the following keys: num_layers,
+      num_nodes, batch_size, num_conv, conv_len, num_classes, learning_rate.
+
+  Returns:
+    ModelFnOps for Estimator API.
+  """
+
+  def _get_input_tensors(features, labels):
+    """Converts the input dict into inks, lengths, and labels tensors."""
+    # features[ink] is a sparse tensor that is [8, batch_maxlen, 3]
+    # inks will be a dense tensor of [8, maxlen, 3]
+    # shapes is [batchsize, 2]
+    shapes = features["shape"]
+    # lengths will be [batch_size]
+    lengths = tf.squeeze(
+        tf.slice(shapes, begin=[0, 0], size=[params.batch_size, 1]))
+    inks = tf.reshape(features["ink"], [params.batch_size, -1, 3])
+    if labels is not None:
+      labels = tf.squeeze(labels)
+    return inks, lengths, labels
+
+  def _add_conv_layers(inks, lengths):
+    """Adds convolution layers."""
+    convolved = inks
+    for i in range(len(params.num_conv)):
+      convolved_input = convolved
+      if params.batch_norm:
+        convolved_input = tf.layers.batch_normalization(
+            convolved_input,
+            training=(mode == tf.estimator.ModeKeys.TRAIN))
+      # Add dropout layer if enabled and not first convolution layer.
+      if i > 0 and params.dropout:
+        convolved_input = tf.layers.dropout(
+            convolved_input,
+            rate=params.dropout,
+            training=(mode == tf.estimator.ModeKeys.TRAIN))
+      convolved = tf.layers.conv1d(
+          convolved_input,
+          filters=params.num_conv[i],
+          kernel_size=params.conv_len[i],
+          activation=None,
+          strides=1,
+          padding="same",
+          name="conv1d_%d" % i)
+    return convolved, lengths
+
+  def _add_regular_rnn_layers(convolved, lengths):
+    """Adds RNN layers."""
+    if params.cell_type == "lstm":
+      cell = tf.nn.rnn_cell.BasicLSTMCell
+    elif params.cell_type == "block_lstm":
+      cell = tf.contrib.rnn.LSTMBlockCell
+    cells_fw = [cell(params.num_nodes) for _ in range(params.num_layers)]
+    cells_bw = [cell(params.num_nodes) for _ in range(params.num_layers)]
+    if params.dropout > 0.0:
+      cells_fw = [tf.contrib.rnn.DropoutWrapper(cell) for cell in cells_fw]
+      cells_bw = [tf.contrib.rnn.DropoutWrapper(cell) for cell in cells_bw]
+    outputs, _, _ = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(
+        cells_fw=cells_fw,
+        cells_bw=cells_bw,
+        inputs=convolved,
+        sequence_length=lengths,
+        dtype=tf.float32,
+        scope="rnn_classification")
+    return outputs
+
+  def _add_cudnn_rnn_layers(convolved):
+    """Adds CUDNN LSTM layers."""
+    # Convolutions output [B, L, Ch], while CudnnLSTM is time-major.
+    convolved = tf.transpose(convolved, [1, 0, 2])
+    lstm = tf.contrib.cudnn_rnn.CudnnLSTM(
+        num_layers=params.num_layers,
+        num_units=params.num_nodes,
+        dropout=params.dropout if mode == tf.estimator.ModeKeys.TRAIN else 0.0,
+        direction="bidirectional")
+    outputs, _ = lstm(convolved)
+    # Convert back from time-major outputs to batch-major outputs.
+    outputs = tf.transpose(outputs, [1, 0, 2])
+    return outputs
+
+  def _add_rnn_layers(convolved, lengths):
+    """Adds recurrent neural network layers depending on the cell type."""
+    if params.cell_type != "cudnn_lstm":
+      outputs = _add_regular_rnn_layers(convolved, lengths)
+    else:
+      outputs = _add_cudnn_rnn_layers(convolved)
+    # outputs is [batch_size, L, N] where L is the maximal sequence length and N
+    # the number of nodes in the last layer.
+    mask = tf.tile(
+        tf.expand_dims(tf.sequence_mask(lengths, tf.shape(outputs)[1]), 2),
+        [1, 1, tf.shape(outputs)[2]])
+    zero_outside = tf.where(mask, outputs, tf.zeros_like(outputs))
+    outputs = tf.reduce_sum(zero_outside, axis=1)
+    return outputs
+
+  def _add_fc_layers(final_state):
+    """Adds a fully connected layer."""
+    return tf.layers.dense(final_state, params.num_classes)
+
+  # Build the model.
+  inks, lengths, labels = _get_input_tensors(features, labels)
+  convolved, lengths = _add_conv_layers(inks, lengths)
+  final_state = _add_rnn_layers(convolved, lengths)
+  logits = _add_fc_layers(final_state)
+  # Add the loss.
+  cross_entropy = tf.reduce_mean(
+      tf.nn.sparse_softmax_cross_entropy_with_logits(
+          labels=labels, logits=logits))
+  # Add the optimizer.
+  train_op = tf.contrib.layers.optimize_loss(
+      loss=cross_entropy,
+      global_step=tf.train.get_global_step(),
+      learning_rate=params.learning_rate,
+      optimizer="Adam",
+      # some gradient clipping stabilizes training in the beginning.
+      clip_gradients=params.gradient_clipping_norm,
+      summaries=["learning_rate", "loss", "gradients", "gradient_norm"])
+  # Compute current predictions.
+  predictions = tf.argmax(logits, axis=1)
+  return tf.estimator.EstimatorSpec(
+      mode=mode,
+      predictions={"logits": logits, "predictions": predictions},
+      loss=cross_entropy,
+      train_op=train_op,
+      eval_metric_ops={"accuracy": tf.metrics.accuracy(labels, predictions)})
+
+
+def create_estimator_and_specs(run_config):
+  """Creates an Experiment configuration based on the estimator and input fn."""
+  model_params = tf.contrib.training.HParams(
+      num_layers=FLAGS.num_layers,
+      num_nodes=FLAGS.num_nodes,
+      batch_size=FLAGS.batch_size,
+      num_conv=ast.literal_eval(FLAGS.num_conv),
+      conv_len=ast.literal_eval(FLAGS.conv_len),
+      num_classes=get_num_classes(),
+      learning_rate=FLAGS.learning_rate,
+      gradient_clipping_norm=FLAGS.gradient_clipping_norm,
+      cell_type=FLAGS.cell_type,
+      batch_norm=FLAGS.batch_norm,
+      dropout=FLAGS.dropout)
+
+  estimator = tf.estimator.Estimator(
+      model_fn=model_fn,
+      config=run_config,
+      params=model_params)
+
+  train_spec = tf.estimator.TrainSpec(input_fn=get_input_fn(
+      mode=tf.estimator.ModeKeys.TRAIN,
+      tfrecord_pattern=FLAGS.training_data,
+      batch_size=FLAGS.batch_size), max_steps=FLAGS.steps)
+
+  eval_spec = tf.estimator.EvalSpec(input_fn=get_input_fn(
+      mode=tf.estimator.ModeKeys.EVAL,
+      tfrecord_pattern=FLAGS.eval_data,
+      batch_size=FLAGS.batch_size))
+
+  return estimator, train_spec, eval_spec
+
+
+def main(unused_args):
+  estimator, train_spec, eval_spec = create_estimator_and_specs(
+      run_config=tf.estimator.RunConfig(
+          model_dir=FLAGS.model_dir,
+          save_checkpoints_secs=300,
+          save_summary_steps=100))
+  tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.register("type", "bool", lambda v: v.lower() == "true")
+  parser.add_argument(
+      "--training_data",
+      type=str,
+      default="",
+      help="Path to training data (tf.Example in TFRecord format)")
+  parser.add_argument(
+      "--eval_data",
+      type=str,
+      default="",
+      help="Path to evaluation data (tf.Example in TFRecord format)")
+  parser.add_argument(
+      "--classes_file",
+      type=str,
+      default="",
+      help="Path to a file with the classes - one class per line")
+  parser.add_argument(
+      "--num_layers",
+      type=int,
+      default=3,
+      help="Number of recurrent neural network layers.")
+  parser.add_argument(
+      "--num_nodes",
+      type=int,
+      default=128,
+      help="Number of node per recurrent network layer.")
+  parser.add_argument(
+      "--num_conv",
+      type=str,
+      default="[48, 64, 96]",
+      help="Number of conv layers along with number of filters per layer.")
+  parser.add_argument(
+      "--conv_len",
+      type=str,
+      default="[5, 5, 3]",
+      help="Length of the convolution filters.")
+  parser.add_argument(
+      "--cell_type",
+      type=str,
+      default="lstm",
+      help="Cell type used for rnn layers: cudnn_lstm, lstm or block_lstm.")
+  parser.add_argument(
+      "--batch_norm",
+      type="bool",
+      default="False",
+      help="Whether to enable batch normalization or not.")
+  parser.add_argument(
+      "--learning_rate",
+      type=float,
+      default=0.0001,
+      help="Learning rate used for training.")
+  parser.add_argument(
+      "--gradient_clipping_norm",
+      type=float,
+      default=9.0,
+      help="Gradient clipping norm used during training.")
+  parser.add_argument(
+      "--dropout",
+      type=float,
+      default=0.3,
+      help="Dropout used for convolutions and bidi lstm layers.")
+  parser.add_argument(
+      "--steps",
+      type=int,
+      default=100000,
+      help="Number of training steps.")
+  parser.add_argument(
+      "--batch_size",
+      type=int,
+      default=8,
+      help="Batch size to use for training/evaluation.")
+  parser.add_argument(
+      "--model_dir",
+      type=str,
+      default="",
+      help="Path for storing the model checkpoints.")
+  parser.add_argument(
+      "--self_test",
+      type="bool",
+      default="False",
+      help="Whether to enable batch normalization or not.")
+
+  FLAGS, unparsed = parser.parse_known_args()
+  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)