Skip to content

GitLab

Unverified Commit 15b33360 authored by Mark Daoust's avatar Mark Daoust Committed by GitHub
Browse files

Merge pull request #4839 from MarkDaoust/autograph 

Add autograph keras layers example
parents a9ab974a 4a897878
Hide whitespace changes
Inline Side-by-side
Showing with 309 additions and 61 deletions
samples/core/guide/autograph.ipynb
View file @ 15b33360
...@@ -113,7 +113,7 @@ ...@@ -113,7 +113,7 @@
}, },
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"! pip install tf-nightly" "! pip install -U tf-nightly"
], ],
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
...@@ -139,8 +139,11 @@ ...@@ -139,8 +139,11 @@
"from __future__ import division, print_function, absolute_import\n", "from __future__ import division, print_function, absolute_import\n",
"\n", "\n",
"import tensorflow as tf\n", "import tensorflow as tf\n",
"import tensorflow.keras.layers as layers\n",
"from tensorflow.contrib import autograph\n", "from tensorflow.contrib import autograph\n",
"\n", "\n",
"\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt" "import matplotlib.pyplot as plt"
], ],
"execution_count": 0, "execution_count": 0,
...@@ -188,7 +191,11 @@ ...@@ -188,7 +191,11 @@
"source": [ "source": [
"## Automatically convert Python control flow\n", "## Automatically convert Python control flow\n",
"\n", "\n",
"AutoGraph will convert much of the Python language into the equivalent TensorFlow graph building code. It converts a function like:" "AutoGraph will convert much of the Python language into the equivalent TensorFlow graph building code. \n",
"\n",
"Note: In real applications batching is essential for performance. The best code to convert to AutoGraph is code where the control flow is decided at the _batch_ level. If making decisions at the individual _example_ level, you must index and batch the examples to maintain performance while applying the control flow logic. \n",
"\n",
"AutoGraph converts a function like:"
] ]
}, },
{ {
...@@ -199,7 +206,7 @@ ...@@ -199,7 +206,7 @@
}, },
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"def g(x):\n", "def square_if_positive(x):\n",
" if x > 0:\n", " if x > 0:\n",
" x = x * x\n", " x = x * x\n",
" else:\n", " else:\n",
...@@ -227,7 +234,7 @@ ...@@ -227,7 +234,7 @@
}, },
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"print(autograph.to_code(g))" "print(autograph.to_code(square_if_positive))"
], ],
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
...@@ -250,7 +257,8 @@ ...@@ -250,7 +257,8 @@
}, },
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"print('Eager results: %2.2f, %2.2f' % (g(tf.constant(9.0)), g(tf.constant(-9.0))))" "print('Eager results: %2.2f, %2.2f' % (square_if_positive(tf.constant(9.0)), \n",
" square_if_positive(tf.constant(-9.0))))"
], ],
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
...@@ -273,13 +281,13 @@ ...@@ -273,13 +281,13 @@
}, },
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"tf_g = autograph.to_graph(g)\n", "tf_square_if_positive = autograph.to_graph(square_if_positive)\n",
"\n", "\n",
"with tf.Graph().as_default(): \n", "with tf.Graph().as_default(): \n",
" # The result works like a regular op: takes tensors in, returns tensors.\n", " # The result works like a regular op: takes tensors in, returns tensors.\n",
" # You can inspect the graph using tf.get_default_graph().as_graph_def()\n", " # You can inspect the graph using tf.get_default_graph().as_graph_def()\n",
" g_out1 = tf_g(tf.constant( 9.0))\n", " g_out1 = tf_square_if_positive(tf.constant( 9.0))\n",
" g_out2 = tf_g(tf.constant(-9.0))\n", " g_out2 = tf_square_if_positive(tf.constant(-9.0))\n",
" with tf.Session() as sess:\n", " with tf.Session() as sess:\n",
" print('Graph results: %2.2f, %2.2f\\n' % (sess.run(g_out1), sess.run(g_out2)))" " print('Graph results: %2.2f, %2.2f\\n' % (sess.run(g_out1), sess.run(g_out2)))"
], ],
...@@ -305,21 +313,20 @@ ...@@ -305,21 +313,20 @@
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"# Continue in a loop\n", "# Continue in a loop\n",
"def f(l):\n", "def sum_even(items):\n",
" s = 0\n", " s = 0\n",
" for c in l:\n", " for c in items:\n",
" if c % 2 > 0:\n", " if c % 2 > 0:\n",
" continue\n", " continue\n",
" s += c\n", " s += c\n",
" return s\n", " return s\n",
"\n", "\n",
"print('Eager result: %d' % f(tf.constant([10,12,15,20])))\n", "print('Eager result: %d' % sum_even(tf.constant([10,12,15,20])))\n",
"\n", "\n",
"tf_f = autograph.to_graph(f)\n", "tf_sum_even = autograph.to_graph(sum_even)\n",
"\n", "\n",
"with tf.Graph().as_default(): \n", "with tf.Graph().as_default(), tf.Session() as sess:\n",
" with tf.Session():\n", " print('Graph result: %d\\n\\n' % sess.run(tf_sum_even(tf.constant([10,12,15,20]))))"
" print('Graph result: %d\\n\\n' % tf_f(tf.constant([10,12,15,20])).eval())"
], ],
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
...@@ -332,7 +339,7 @@ ...@@ -332,7 +339,7 @@
}, },
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"print(autograph.to_code(f))" "print(autograph.to_code(sum_even))"
], ],
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
...@@ -358,26 +365,26 @@ ...@@ -358,26 +365,26 @@
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"@autograph.convert()\n", "@autograph.convert()\n",
"def fizzbuzz(num):\n", "def fizzbuzz(i, n):\n",
" if num % 3 == 0 and num % 5 == 0:\n", " while i < n:\n",
" print('FizzBuzz')\n", " msg = ''\n",
" elif num % 3 == 0:\n", " if i % 3 == 0:\n",
" print('Fizz')\n", " msg += 'Fizz'\n",
" elif num % 5 == 0:\n", " if i % 5 == 0:\n",
" print('Buzz')\n", " msg += 'Buzz'\n",
" else:\n", " if msg == '':\n",
" print(num)\n", " msg = tf.as_string(i)\n",
" return num\n", " print(msg)\n",
"\n", " i += 1\n",
" return i\n",
"\n", "\n",
"with tf.Graph().as_default():\n", "with tf.Graph().as_default():\n",
" final_i = fizzbuzz(tf.constant(10), tf.constant(16))\n",
" # The result works like a regular op: takes tensors in, returns tensors.\n", " # The result works like a regular op: takes tensors in, returns tensors.\n",
" # You can inspect the graph using tf.get_default_graph().as_graph_def()\n", " # You can inspect the graph using tf.get_default_graph().as_graph_def()\n",
" num = tf.placeholder(tf.int32)\n",
" result = fizzbuzz(num)\n",
" with tf.Session() as sess:\n", " with tf.Session() as sess:\n",
" for n in range(10,16):\n", " sess.run(final_i)\n",
" sess.run(result, feed_dict={num:n})" "\n"
], ],
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
...@@ -391,7 +398,7 @@ ...@@ -391,7 +398,7 @@
"source": [ "source": [
"## Examples\n", "## Examples\n",
"\n", "\n",
"Let's demonstrate some useful Python language features." "Let's demonstrate some useful Python language features.\n"
] ]
}, },
{ {
...@@ -415,16 +422,15 @@ ...@@ -415,16 +422,15 @@
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"@autograph.convert()\n", "@autograph.convert()\n",
"def f(x):\n", "def inverse(x):\n",
" assert x != 0, 'Do not pass zero!'\n", " assert x != 0.0, 'Do not pass zero!'\n",
" return x * x\n", " return 1.0 / x\n",
"\n", "\n",
"with tf.Graph().as_default(): \n", "with tf.Graph().as_default(), tf.Session() as sess:\n",
" with tf.Session():\n", " try:\n",
" try:\n", " print(sess.run(inverse(tf.constant(0.0))))\n",
" print(f(tf.constant(0)).eval())\n", " except tf.errors.InvalidArgumentError as e:\n",
" except tf.errors.InvalidArgumentError as e:\n", " print('Got error message:\\n %s' % e.message)"
" print('Got error message:\\n %s' % e.message)"
], ],
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
...@@ -443,30 +449,29 @@ ...@@ -443,30 +449,29 @@
}, },
{ {
"metadata": { "metadata": {
"id": "ySTsuxnqCTQi", "id": "ehBac9rUR6nh",
"colab_type": "code", "colab_type": "code",
"colab": {} "colab": {}
}, },
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"@autograph.convert()\n", "@autograph.convert()\n",
"def f(n):\n", "def count(n):\n",
" if n >= 0:\n", " i=0\n",
" while n < 5:\n", " while i < n:\n",
" n += 1\n", " print(i)\n",
" print(n)\n", " i += 1\n",
" return n\n", " return n\n",
" \n", " \n",
"with tf.Graph().as_default():\n", "with tf.Graph().as_default(), tf.Session() as sess:\n",
" with tf.Session():\n", " sess.run(count(tf.constant(5)))"
" f(tf.constant(0)).eval()"
], ],
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
}, },
{ {
"metadata": { "metadata": {
"id": "NqF0GT-VCVFh", "id": "mtpegD_YR6HK",
"colab_type": "text" "colab_type": "text"
}, },
"cell_type": "markdown", "cell_type": "markdown",
...@@ -485,7 +490,7 @@ ...@@ -485,7 +490,7 @@
"cell_type": "code", "cell_type": "code",
"source": [ "source": [
"@autograph.convert()\n", "@autograph.convert()\n",
"def f(n):\n", "def arange(n):\n",
" z = []\n", " z = []\n",
" # We ask you to tell us the element dtype of the list\n", " # We ask you to tell us the element dtype of the list\n",
" autograph.set_element_type(z, tf.int32)\n", " autograph.set_element_type(z, tf.int32)\n",
...@@ -496,11 +501,9 @@ ...@@ -496,11 +501,9 @@
" # (this is just like np.stack)\n", " # (this is just like np.stack)\n",
" return autograph.stack(z) \n", " return autograph.stack(z) \n",
"\n", "\n",
"#tf_f = autograph.to_graph(f)\n",
"\n", "\n",
"with tf.Graph().as_default(): \n", "with tf.Graph().as_default(), tf.Session() as sess:\n",
" with tf.Session():\n", " sess.run(arange(tf.constant(10)))"
" print(f(tf.constant(3)).eval())"
], ],
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
...@@ -512,7 +515,7 @@ ...@@ -512,7 +515,7 @@
}, },
"cell_type": "markdown", "cell_type": "markdown",
"source": [ "source": [
"### Nested if statements" "### Nested control flow"
] ]
}, },
{ {
...@@ -571,6 +574,42 @@ ...@@ -571,6 +574,42 @@
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
}, },
{
"metadata": {
"id": "3N1mz7sNY87N",
"colab_type": "text"
},
"cell_type": "markdown",
"source": [
"### For loop"
]
},
{
"metadata": {
"id": "CFk2fszrY8af",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"@autograph.convert()\n",
"def fizzbuzz_each(nums):\n",
"\n",
" result = []\n",
" autograph.set_element_type(result, tf.string)\n",
"\n",
" for num in nums: \n",
" result.append(fizzbuzz(num))\n",
" \n",
" return autograph.stack(result)\n",
" \n",
"with tf.Graph().as_default(): \n",
" with tf.Session() as sess:\n",
" print(sess.run(fizzbuzz_each(tf.constant(np.arange(10)))))"
],
"execution_count": 0,
"outputs": []
},
{ {
"metadata": { "metadata": {
"id": "FXB0Zbwl13PY", "id": "FXB0Zbwl13PY",
...@@ -609,6 +648,217 @@ ...@@ -609,6 +648,217 @@
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
}, },
{
"metadata": {
"id": "XY4UspHmZNdL",
"colab_type": "text"
},
"cell_type": "markdown",
"source": [
"## Interoperation with `tf.Keras`\n",
"\n",
"Now that you've seen the basics, let's build some model components with autograph.\n",
"\n",
"It's relatively simple to integrate `autograph` with `tf.keras`. \n",
"\n",
"\n",
"### Stateless functions\n",
"\n",
"For stateless functions, like `collatz` shown below, the easiest way to include them in a keras model is to wrap them up as a layer uisng `tf.keras.layers.Lambda`."
]
},
{
"metadata": {
"id": "ChZh3q-zcF6C",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"import numpy as np\n",
"\n",
"@autograph.convert()\n",
"def collatz(x):\n",
" x=tf.reshape(x,())\n",
" assert x>0\n",
" n = tf.convert_to_tensor((0,)) \n",
" while not tf.equal(x,1):\n",
" n+=1\n",
" if tf.equal(x%2, 0):\n",
" x = x//2\n",
" else:\n",
" x = 3*x+1\n",
" \n",
" return n\n",
"\n",
"with tf.Graph().as_default():\n",
" model = tf.keras.Sequential([\n",
" tf.keras.layers.Lambda(collatz, input_shape=(1,), output_shape=(), )\n",
" ])\n",
" \n",
"result = model.predict(np.array([6171])) #261\n",
"result"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "k9LEoa3ud9hA",
"colab_type": "text"
},
"cell_type": "markdown",
"source": [
"### Custom Layers and Models\n",
"\n",
"<!--TODO(markdaoust) link to full examples or these referenced models.-->\n",
"\n",
"The easiest way to use AutoGraph with Keras layers and models is to `@autograph.convert()` the `call` method. See the [TensorFlow Keras guide](https://tensorflow.org/guide/keras#build_advanced_models) for details on how to build on these classes. \n",
"\n",
"Here is a simple example of the [stocastic network depth](https://arxiv.org/abs/1603.09382) technique :"
]
},
{
"metadata": {
"id": "DJi_RJkeeOju",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"# `K` is used to check if we're in train or test mode.\n",
"import tensorflow.keras.backend as K\n",
"\n",
"class StocasticNetworkDepth(tf.keras.Sequential):\n",
" def __init__(self, pfirst=1.0, plast=0.5, *args,**kwargs):\n",
" self.pfirst = pfirst\n",
" self.plast = plast\n",
" super().__init__(*args,**kwargs)\n",
" \n",
" def build(self,input_shape):\n",
" super().build(input_shape.as_list())\n",
" self.depth = len(self.layers)\n",
" self.plims = np.linspace(self.pfirst, self.plast, self.depth+1)[:-1]\n",
" \n",
" @autograph.convert()\n",
" def call(self, inputs):\n",
" training = tf.cast(K.learning_phase(), dtype=bool) \n",
" if not training: \n",
" count = self.depth\n",
" return super(StocasticNetworkDepth, self).call(inputs), count\n",
" \n",
" p = tf.random_uniform((self.depth,))\n",
" \n",
" keeps = p<=self.plims\n",
" x = inputs\n",
" \n",
" count = tf.reduce_sum(tf.cast(keeps, tf.int32))\n",
" for i in range(self.depth):\n",
" if keeps[i]:\n",
" x = self.layers[i](x)\n",
" \n",
" # return both the final-layer output and the number of layers executed.\n",
" return x, count"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "NIEzuNL6vMVl",
"colab_type": "text"
},
"cell_type": "markdown",
"source": [
"Let's try it on mnist-shaped data:"
]
},
{
"metadata": {
"id": "FiqyFySkWbeN",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"train_batch = np.random.randn(64, 28,28,1).astype(np.float32)"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "Vz1JTpLOvT4u",
"colab_type": "text"
},
"cell_type": "markdown",
"source": [
"Build a simple stack of `conv` layers, in the stocastic depth model:"
]
},
{
"metadata": {
"id": "XwwtlQAjvUph",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"with tf.Graph().as_default() as g:\n",
" model = StocasticNetworkDepth(\n",
" pfirst=1.0, plast=0.5)\n",
"\n",
" for n in range(20):\n",
" model.add(\n",
" layers.Conv2D(filters=16, activation=tf.nn.relu,\n",
" kernel_size=(3,3), padding='same'))\n",
"\n",
" model.build(tf.TensorShape((None, None, None,1)))\n",
" \n",
" init = tf.global_variables_initializer()"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "uM3g_v7mvrkg",
"colab_type": "text"
},
"cell_type": "markdown",
"source": [
"Now test it to ensure it behaves as expected in train and test modes:"
]
},
{
"metadata": {
"id": "7tdmuh5Zvm3D",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"# Use an explicit session here so we can set the train/test switch, and\n",
"# inspect the layer count returned by `call`\n",
"with tf.Session(graph=g) as sess:\n",
" init.run()\n",
" \n",
" for phase, name in enumerate(['test','train']):\n",
" K.set_learning_phase(phase)\n",
" result, count = model(tf.convert_to_tensor(train_batch, dtype=tf.float32))\n",
"\n",
" result1, count1 = sess.run((result, count))\n",
" result2, count2 = sess.run((result, count))\n",
"\n",
" delta = (result1 - result2)\n",
" print(name, \"sum abs delta: \", abs(delta).mean())\n",
" print(\" layers 1st call: \", count1)\n",
" print(\" layers 2nd call: \", count2)\n",
" print()"
],
"execution_count": 0,
"outputs": []
},
{ {
"metadata": { "metadata": {
"id": "4LfnJjm0Bm0B", "id": "4LfnJjm0Bm0B",
...@@ -618,11 +868,9 @@ ...@@ -618,11 +868,9 @@
"source": [ "source": [
"## Advanced example: An in-graph training loop\n", "## Advanced example: An in-graph training loop\n",
"\n", "\n",
"Since writing control flow in AutoGraph is easy, running a training loop in a TensorFlow graph should also be easy. \n", "The previous section showed that AutoGraph can be used inside Keras layers and models. Keras models can also be used in AutoGraph code.\n",
"\n", "\n",
"<!--TODO(markdaoust) link to examples showing autograph **in** keras models when ready-->\n", "Since writing control flow in AutoGraph is easy, running a training loop in a TensorFlow graph should also be easy. \n",
"\n",
"Important: While this example wraps a `tf.keras.Model` using AutoGraph, `tf.contrib.autograph` is compatible with `tf.keras` and can be used in [Keras custom layers and models](https://tensorflow.org/guide/keras#build_advanced_models). The easiest way is to `@autograph.convert()` the `call` method.\n",
"\n", "\n",
"This example shows how to train a simple Keras model on MNIST with the entire training process—loading batches, calculating gradients, updating parameters, calculating validation accuracy, and repeating until convergence—is performed in-graph." "This example shows how to train a simple Keras model on MNIST with the entire training process—loading batches, calculating gradients, updating parameters, calculating validation accuracy, and repeating until convergence—is performed in-graph."
] ]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment