Commit d50165de authored by Mark Daoust's avatar Mark Daoust
Browse files

Remove incomplete RNN example.

parent bfee3a1f
...@@ -711,7 +711,7 @@ ...@@ -711,7 +711,7 @@
"\n", "\n",
"<!--TODO(markdaoust) link to full examples or these referenced models.-->\n", "<!--TODO(markdaoust) link to full examples or these referenced models.-->\n",
"\n", "\n",
"The easiest way is to `@autograph.convert()` the `call` method. See the [keras guide](https://tensorflow.org/guide/keras#build_advanced_models) for details on how to build on these classes. \n", "The easiest way to use autograph is keras layers and models is to `@autograph.convert()` the `call` method. See the [keras guide](https://tensorflow.org/guide/keras#build_advanced_models) for details on how to build on these classes. \n",
"\n", "\n",
"Here is a simple example of the [stocastic network depth](https://arxiv.org/abs/1603.09382) technique :" "Here is a simple example of the [stocastic network depth](https://arxiv.org/abs/1603.09382) technique :"
] ]
...@@ -857,102 +857,6 @@ ...@@ -857,102 +857,6 @@
"execution_count": 0, "execution_count": 0,
"outputs": [] "outputs": []
}, },
{
"metadata": {
"id": "cpUD21HQWcOq",
"colab_type": "text"
},
"cell_type": "markdown",
"source": [
"### RNN Cells\n",
"\n",
"The [standard approach](https://www.tensorflow.org/api_docs/python/tf/keras/layers/RNN) to custom RNN cells has the same issues that are solved by autograph.\n",
"\n",
"Implementing RNN cells with `autograph` is not much different from implementing them [under eager execution](https://colab.sandbox.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb).\n",
"\n",
"To implement the prediction step in a keras model you could say:\n",
"\n"
]
},
{
"metadata": {
"id": "798S1r-sJGfR",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"class BahdanauAttention(tf.keras.Model):\n",
" def __init__(self, units):\n",
" super(BahdanauAttention, self).__init__()\n",
" self.W1 = tf.keras.layers.Dense(units)\n",
" self.W2 = tf.keras.layers.Dense(units)\n",
" self.V = tf.keras.layers.Dense(1)\n",
" \n",
" def call(self, features, hidden):\n",
" hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
" score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n",
" attention_weights = tf.nn.softmax(self.V(score), axis=1)\n",
" context_vector = attention_weights * features\n",
" context_vector = tf.reduce_sum(context_vector, axis=1)\n",
" return context_vector, attention_weights"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "qwH-QnmlGV6c",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"class Decoder(tf.keras.Model):\n",
" def __init__(self, vocab_size, embedding_dim, dec_units):\n",
" super(Decoder, self).__init__()\n",
" self.dec_units = dec_units\n",
" self.embedding = layers.Embedding(vocab_size, embedding_dim)\n",
" self.gru = layers.GRU(self.dec_units)\n",
" self.fc = tf.keras.layers.Dense(vocab_size)\n",
" self.attention = BahdanauAttention(self.dec_units)\n",
" \n",
" def call(self, enc_output):\n",
" results = tf.keras\n",
" hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
" score = tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis))\n",
" \n",
" # attention_weights shape == (batch_size, max_length, 1)\n",
" # we get 1 at the last axis because we are applying score to self.V\n",
" attention_weights = tf.nn.softmax(self.V(score), axis=1)\n",
" \n",
" # context_vector shape after sum == (batch_size, hidden_size)\n",
" context_vector = attention_weights * enc_output\n",
" context_vector = tf.reduce_sum(context_vector, axis=1)\n",
" \n",
" # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
" x = self.embedding(x)\n",
" \n",
" # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
" x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
" \n",
" # passing the concatenated vector to the GRU\n",
" output, state = self.gru(x)\n",
" \n",
" # output shape == (batch_size * max_length, hidden_size)\n",
" output = tf.reshape(output, (-1, output.shape[2]))\n",
" \n",
" # output shape == (batch_size * max_length, vocab)\n",
" x = self.fc(output)\n",
" \n",
" return x, state, attention_weights\n",
" \n",
" def initialize_hidden_state(self):\n",
" return tf.zeros((self.batch_sz, self.dec_units))"
],
"execution_count": 0,
"outputs": []
},
{ {
"metadata": { "metadata": {
"id": "4LfnJjm0Bm0B", "id": "4LfnJjm0Bm0B",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment