Generate_Bible_ish_verses.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Generate Bible-ish verses.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "yG_n40gFzf9s",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass\n",
        "import tensorflow as tf\n",
        "\n",
        "import numpy as np\n",
        "import os\n",
        "import time"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "pD_55cOxLkAb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "2db7d6f5-2700-4161-8d0e-cece76798fa1"
      },
      "source": [
        "path_to_file = tf.keras.utils.get_file('bible.txt', 'http://www.gutenberg.org/cache/epub/10/pg10.txt')"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from http://www.gutenberg.org/cache/epub/10/pg10.txt\n",
            "4456448/4452519 [==============================] - 2s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "aavnuByVymwK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f3ef8740-357c-4ed5-b2db-e80e6cc86e11"
      },
      "source": [
        "# Read, then decode for py2 compat.\n",
        "text = open(path_to_file, 'rb').read().decode(encoding='utf-8')\n",
        "# length of text is the number of characters in it\n",
        "print ('Length of text: {} characters'.format(len(text)))"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Length of text: 4452517 characters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Duhg9NrUymwO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        },
        "outputId": "c195169a-69b5-4e76-bfbf-a13cb8a90748"
      },
      "source": [
        "# Take a look at the first 250 characters in text\n",
        "print(text[:250])"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "﻿The Project Gutenberg EBook of The King James Bible\r\n",
            "\r\n",
            "\r\n",
            "**********************************************************************\r\n",
            "EBOOK (#10) WAS ONE OF PROJECT GUTENBERG'S EARLY FILES PRODUCED AT A\r\n",
            "TIME WHEN PROOFING METHODS AND TOOLS WERE NOT WELL\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "IlCgQBRVymwR",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0d2b2f4c-02c5-45de-90b6-3c11cddc2209"
      },
      "source": [
        "# The unique characters in the file\n",
        "vocab = sorted(set(text))\n",
        "print ('{} unique characters'.format(len(vocab)))"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "85 unique characters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "IalZLbvOzf-F",
        "colab": {}
      },
      "source": [
        "# Creating a mapping from unique characters to indices\n",
        "char2idx = {u:i for i, u in enumerate(vocab)}\n",
        "idx2char = np.array(vocab)\n",
        "\n",
        "text_as_int = np.array([char2idx[c] for c in text])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "FYyNlCNXymwY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 403
        },
        "outputId": "863d8927-70ed-4b4f-975d-fa173a966bf0"
      },
      "source": [
        "print('{')\n",
        "for char,_ in zip(char2idx, range(20)):\n",
        "    print('  {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n",
        "print('  ...\\n}')"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{\n",
            "  '\\n':   0,\n",
            "  '\\r':   1,\n",
            "  ' ' :   2,\n",
            "  '!' :   3,\n",
            "  '\"' :   4,\n",
            "  '#' :   5,\n",
            "  '$' :   6,\n",
            "  '%' :   7,\n",
            "  \"'\" :   8,\n",
            "  '(' :   9,\n",
            "  ')' :  10,\n",
            "  '*' :  11,\n",
            "  ',' :  12,\n",
            "  '-' :  13,\n",
            "  '.' :  14,\n",
            "  '/' :  15,\n",
            "  '0' :  16,\n",
            "  '1' :  17,\n",
            "  '2' :  18,\n",
            "  '3' :  19,\n",
            "  ...\n",
            "}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "l1VKcQHcymwb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8070c105-b0c3-46b3-ff09-82e2933959dd"
      },
      "source": [
        "# Show how the first 13 characters from the text are mapped to integers\n",
        "print ('{} ---- characters mapped to int ---- > {}'.format(repr(text[:13]), text_as_int[:13]))"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "'\\ufeffThe Project ' ---- characters mapped to int ---- > [84 49 65 62  2 45 75 72 67 62 60 77  2]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0UHJDA39zf-O",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        },
        "outputId": "a93923ce-b86b-4d89-94e6-24cb16b9a4d7"
      },
      "source": [
        "# The maximum length sentence we want for a single input in characters\n",
        "seq_length = 100\n",
        "examples_per_epoch = len(text)//(seq_length+1)\n",
        "\n",
        "# Create training examples / targets\n",
        "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n",
        "\n",
        "for i in char_dataset.take(5):\n",
        "  print(idx2char[i.numpy()])"
      ],
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "﻿\n",
            "T\n",
            "h\n",
            "e\n",
            " \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "l4hkDU3i7ozi",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        },
        "outputId": "e5b979be-2c94-4240-a2ef-cd5063793064"
      },
      "source": [
        "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n",
        "\n",
        "for item in sequences.take(5):\n",
        "  print(repr(''.join(idx2char[item.numpy()])))"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "'\\ufeffThe Project Gutenberg EBook of The King James Bible\\r\\n\\r\\n\\r\\n*******************************************'\n",
            "\"***************************\\r\\nEBOOK (#10) WAS ONE OF PROJECT GUTENBERG'S EARLY FILES PRODUCED AT A\\r\\nTI\"\n",
            "'ME WHEN PROOFING METHODS AND TOOLS WERE NOT WELL DEVELOPED. THERE IS\\r\\nAN IMPROVED EDITION OF THIS TIT'\n",
            "'LE WHICH MAY VIEWED AT EBOOK http://www.gutenberg.org/files/10900/10900-h/10900-h.htm\\r\\n(There is no t'\n",
            "'ext file for this ebook)\\r\\n**********************************************************************\\r\\n\\r\\n\\r'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9NGu-FkO_kYU",
        "colab": {}
      },
      "source": [
        "def split_input_target(chunk):\n",
        "    input_text = chunk[:-1]\n",
        "    target_text = chunk[1:]\n",
        "    return input_text, target_text\n",
        "\n",
        "dataset = sequences.map(split_input_target)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GNbw-iR0ymwj",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "95f9f98d-6c92-4b79-f14a-73e6b36e7826"
      },
      "source": [
        "for input_example, target_example in  dataset.take(1):\n",
        "  print ('Input data: ', repr(''.join(idx2char[input_example.numpy()])))\n",
        "  print ('Target data:', repr(''.join(idx2char[target_example.numpy()])))"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input data:  '\\ufeffThe Project Gutenberg EBook of The King James Bible\\r\\n\\r\\n\\r\\n******************************************'\n",
            "Target data: 'The Project Gutenberg EBook of The King James Bible\\r\\n\\r\\n\\r\\n*******************************************'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0eBu9WZG84i0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "ef88f67c-1904-4f1b-feec-7e3ceaa52764"
      },
      "source": [
        "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n",
        "    print(\"Step {:4d}\".format(i))\n",
        "    print(\"  input: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n",
        "    print(\"  expected output: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Step    0\n",
            "  input: 84 ('\\ufeff')\n",
            "  expected output: 49 ('T')\n",
            "Step    1\n",
            "  input: 49 ('T')\n",
            "  expected output: 65 ('h')\n",
            "Step    2\n",
            "  input: 65 ('h')\n",
            "  expected output: 62 ('e')\n",
            "Step    3\n",
            "  input: 62 ('e')\n",
            "  expected output: 2 (' ')\n",
            "Step    4\n",
            "  input: 2 (' ')\n",
            "  expected output: 45 ('P')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "p2pGotuNzf-S",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "23956f79-e01a-41fe-dd84-4c85107d17b9"
      },
      "source": [
        "# Batch size\n",
        "BATCH_SIZE = 64\n",
        "\n",
        "# Buffer size to shuffle the dataset\n",
        "# (TF data is designed to work with possibly infinite sequences,\n",
        "# so it doesn't attempt to shuffle the entire sequence in memory. Instead,\n",
        "# it maintains a buffer in which it shuffles elements).\n",
        "BUFFER_SIZE = 10000\n",
        "\n",
        "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n",
        "\n",
        "dataset"
      ],
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<BatchDataset shapes: ((64, 100), (64, 100)), types: (tf.int64, tf.int64)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zHT8cLh7EAsg",
        "colab": {}
      },
      "source": [
        "# Length of the vocabulary in chars\n",
        "vocab_size = len(vocab)\n",
        "\n",
        "# The embedding dimension\n",
        "embedding_dim = 256\n",
        "\n",
        "# Number of RNN units\n",
        "rnn_units = 1024"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MtCrdfzEI2N0",
        "colab": {}
      },
      "source": [
        "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n",
        "  model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Embedding(vocab_size, embedding_dim,\n",
        "                              batch_input_shape=[batch_size, None]),\n",
        "    tf.keras.layers.GRU(rnn_units,\n",
        "                        return_sequences=True,\n",
        "                        stateful=True,\n",
        "                        recurrent_initializer='glorot_uniform'),\n",
        "    tf.keras.layers.Dense(vocab_size)\n",
        "  ])\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "wwsrpOik5zhv",
        "colab": {}
      },
      "source": [
        "model = build_model(\n",
        "  vocab_size = len(vocab),\n",
        "  embedding_dim=embedding_dim,\n",
        "  rnn_units=rnn_units,\n",
        "  batch_size=BATCH_SIZE)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "C-_70kKAPrPU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "93d3c8a0-64a7-4db9-d154-c4309da64e3a"
      },
      "source": [
        "for input_example_batch, target_example_batch in dataset.take(1):\n",
        "  example_batch_predictions = model(input_example_batch)\n",
        "  print(example_batch_predictions.shape, \"# (batch_size, sequence_length, vocab_size)\")"
      ],
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(64, 100, 85) # (batch_size, sequence_length, vocab_size)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vPGmAAXmVLGC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252
        },
        "outputId": "4517af04-d914-4da0-e888-9ae197edcc15"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_4\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_4 (Embedding)      (64, None, 256)           21760     \n",
            "_________________________________________________________________\n",
            "gru_4 (GRU)                  (64, None, 1024)          3938304   \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (64, None, 85)            87125     \n",
            "=================================================================\n",
            "Total params: 4,047,189\n",
            "Trainable params: 4,047,189\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4V4MfFg0RQJg",
        "colab": {}
      },
      "source": [
        "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)\n",
        "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "YqFMUQc_UFgM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        },
        "outputId": "3f469c7b-34d1-4307-ddf7-35bdeb4bef0e"
      },
      "source": [
        "sampled_indices"
      ],
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([10, 79, 41, 64, 11, 70,  6, 47,  7,  4, 36, 16, 76, 41, 28, 44, 19,\n",
              "       11, 73, 28, 52,  4, 27, 23, 19,  5, 34, 32, 12, 54, 46, 61, 25, 21,\n",
              "       35, 84, 72, 59, 17, 72, 66, 26, 64, 41, 78, 31, 56, 24, 68, 42, 81,\n",
              "       84, 58, 55, 59, 48, 65, 82, 15, 77,  1, 64, 54, 12, 17, 57, 29, 72,\n",
              "       46, 74, 63, 25,  2, 42, 16, 24, 30,  3, 66, 46, 69, 72, 62, 70, 53,\n",
              "       21, 49, 38, 47,  1, 35, 60, 13, 17, 57, 47, 80, 76, 54, 38])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xWcFwPwLSo05",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        },
        "outputId": "cf176581-3521-4ffe-dc74-95e8ecd2023f"
      },
      "source": [
        "print(\"Input: \\n\", repr(\"\".join(idx2char[input_example_batch[0]])))\n",
        "print()\n",
        "print(\"Next Char Predictions: \\n\", repr(\"\".join(idx2char[sampled_indices ])))"
      ],
      "execution_count": 87,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: \n",
            " 'Pharaoh, Thus\\r\\nsaith the LORD God of Israel, Let my people go, that they may hold a\\r\\nfeast unto me i'\n",
            "\n",
            "Next Char Predictions: \n",
            " ')vLg*m$R%\"G0sL?O3*p?W\";73#EC,YQd95F\\ufeffob1oi:gLuB[8kMx\\ufeffaZbShy/t\\rgY,1]@oQqf9 M08A!iQloemX5TIR\\rFc-1]RwsYI'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4HrXTACTdzY-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "f84656d5-6448-480e-a192-8d8b318853b1"
      },
      "source": [
        "def loss(labels, logits):\n",
        "  return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n",
        "\n",
        "example_batch_loss  = loss(target_example_batch, example_batch_predictions)\n",
        "print(\"Prediction shape: \", example_batch_predictions.shape, \" # (batch_size, sequence_length, vocab_size)\")\n",
        "print(\"scalar_loss:      \", example_batch_loss.numpy().mean())"
      ],
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Prediction shape:  (64, 100, 85)  # (batch_size, sequence_length, vocab_size)\n",
            "scalar_loss:       4.442077\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "DDl1_Een6rL0",
        "colab": {}
      },
      "source": [
        "model.compile(optimizer='adam', loss=loss)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "W6fWTriUZP-n",
        "colab": {}
      },
      "source": [
        "# Directory where the checkpoints will be saved\n",
        "checkpoint_dir = './training_checkpoints'\n",
        "# Name of the checkpoint files\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n",
        "\n",
        "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_prefix,\n",
        "    save_weights_only=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7yGBE2zxMMHs",
        "colab": {}
      },
      "source": [
        "EPOCHS=10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UK-hmKjYVoll",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 370
        },
        "outputId": "bc911218-aedb-48b2-f355-bb246d6f2163"
      },
      "source": [
        "history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train for 688 steps\n",
            "Epoch 1/10\n",
            "688/688 [==============================] - 43s 62ms/step - loss: 1.7337\n",
            "Epoch 2/10\n",
            "688/688 [==============================] - 42s 61ms/step - loss: 1.1657\n",
            "Epoch 3/10\n",
            "688/688 [==============================] - 42s 61ms/step - loss: 1.0777\n",
            "Epoch 4/10\n",
            "688/688 [==============================] - 42s 61ms/step - loss: 1.0330\n",
            "Epoch 5/10\n",
            "688/688 [==============================] - 42s 61ms/step - loss: 1.0026\n",
            "Epoch 6/10\n",
            "688/688 [==============================] - 42s 61ms/step - loss: 0.9788\n",
            "Epoch 7/10\n",
            "688/688 [==============================] - 42s 61ms/step - loss: 0.9593\n",
            "Epoch 8/10\n",
            "688/688 [==============================] - 42s 61ms/step - loss: 0.9426\n",
            "Epoch 9/10\n",
            "688/688 [==============================] - 42s 61ms/step - loss: 0.9290\n",
            "Epoch 10/10\n",
            "688/688 [==============================] - 42s 61ms/step - loss: 0.9181\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zk2WJ2-XjkGz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4600c51f-8758-4558-e7df-bb32ddf966fa"
      },
      "source": [
        "tf.train.latest_checkpoint(checkpoint_dir)"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'./training_checkpoints/ckpt_10'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "LycQ-ot_jjyu",
        "colab": {}
      },
      "source": [
        "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n",
        "\n",
        "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n",
        "\n",
        "model.build(tf.TensorShape([1, None]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "71xa6jnYVrAN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252
        },
        "outputId": "0ed7f132-6f30-4fd0-a674-ee4cd6c4b4c3"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_5\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_5 (Embedding)      (1, None, 256)            21760     \n",
            "_________________________________________________________________\n",
            "gru_5 (GRU)                  (1, None, 1024)           3938304   \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (1, None, 85)             87125     \n",
            "=================================================================\n",
            "Total params: 4,047,189\n",
            "Trainable params: 4,047,189\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WvuwZBX5Ogfd",
        "colab": {}
      },
      "source": [
        "def generate_text(model, start_string):\n",
        "  # Evaluation step (generating text using the learned model)\n",
        "\n",
        "  # Number of characters to generate\n",
        "  num_generate = 1000\n",
        "\n",
        "  # Converting our start string to numbers (vectorizing)\n",
        "  input_eval = [char2idx[s] for s in start_string]\n",
        "  input_eval = tf.expand_dims(input_eval, 0)\n",
        "\n",
        "  # Empty string to store our results\n",
        "  text_generated = []\n",
        "\n",
        "  # Low temperatures results in more predictable text.\n",
        "  # Higher temperatures results in more surprising text.\n",
        "  # Experiment to find the best setting.\n",
        "  temperature = 1.0\n",
        "\n",
        "  # Here batch size == 1\n",
        "  model.reset_states()\n",
        "  for i in range(num_generate):\n",
        "      predictions = model(input_eval)\n",
        "      # remove the batch dimension\n",
        "      predictions = tf.squeeze(predictions, 0)\n",
        "\n",
        "      # using a categorical distribution to predict the word returned by the model\n",
        "      predictions = predictions / temperature\n",
        "      predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n",
        "\n",
        "      # We pass the predicted word as the next input to the model\n",
        "      # along with the previous hidden state\n",
        "      input_eval = tf.expand_dims([predicted_id], 0)\n",
        "\n",
        "      text_generated.append(idx2char[predicted_id])\n",
        "\n",
        "  return (start_string + ''.join(text_generated))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ktovv0RFhrkn",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 319
        },
        "outputId": "6231748c-691e-4e9a-f6e5-029188bd1c28"
      },
      "source": [
        "print(generate_text(model, start_string=u\"GOD: \"))"
      ],
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GOD: But ye are fire.\r\n",
            "\r\n",
            "4:22 Fear my God for Doas, that my soul desireth that he will, and let all natural permission in hindry:\r\n",
            "(for thou hast this man to give you thess: for they said, O thou hast kept,\r\n",
            "as for I bear not, likewise, draw and enquire honour, and of hypocrisy;\r\n",
            "and husbands shall know that he will search for your children: for why doth man\r\n",
            "be a royention oor had gotten in his way.\r\n",
            "\r\n",
            "19:13 And now the daughterst that verith, God of God, for the sin offering: for so\r\n",
            "was I caused in their goings in the blood of righony perceiver.\r\n",
            "\r\n",
            "5:28 Brethren, be the seen of Jesus Christ, that if is John, whose armourbearer\r\n",
            "said, The voice of their hands are suppaised: sin thou not by all, and shall\r\n",
            "blossom, and scame tit his wife unto him the inhabitants of Jerusalem.\r\n",
            "\r\n",
            "7:11 And when he was constrained from Judaea we smite to scail, from the door of the te, when I have sold\r\n",
            "unto the blessing: I beseech thee this fetheclook not one: 13:6\r\n",
            "And shall say to this law house of Judas, \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_XAm7eCoKULT",
        "colab": {}
      },
      "source": [
        "model = build_model(\n",
        "  vocab_size = len(vocab),\n",
        "  embedding_dim=embedding_dim,\n",
        "  rnn_units=rnn_units,\n",
        "  batch_size=BATCH_SIZE)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qUKhnZtMVpoJ",
        "colab": {}
      },
      "source": [
        "optimizer = tf.keras.optimizers.Adam()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "b4kH1o0leVIp",
        "colab": {}
      },
      "source": [
        "@tf.function\n",
        "def train_step(inp, target):\n",
        "  with tf.GradientTape() as tape:\n",
        "    predictions = model(inp)\n",
        "    loss = tf.reduce_mean(\n",
        "        tf.keras.losses.sparse_categorical_crossentropy(\n",
        "            target, predictions, from_logits=True))\n",
        "  grads = tape.gradient(loss, model.trainable_variables)\n",
        "  optimizer.apply_gradients(zip(grads, model.trainable_variables))\n",
        "\n",
        "  return loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "d4tSNwymzf-q",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "17b17b86-e18f-4d5e-c904-bc8cc6f1b345"
      },
      "source": [
        "# Training step\n",
        "EPOCHS = 10\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "  start = time.time()\n",
        "\n",
        "  # initializing the hidden state at the start of every epoch\n",
        "  # initally hidden is None\n",
        "  hidden = model.reset_states()\n",
        "\n",
        "  for (batch_n, (inp, target)) in enumerate(dataset):\n",
        "    loss = train_step(inp, target)\n",
        "\n",
        "    if batch_n % 100 == 0:\n",
        "      template = 'Epoch {} Batch {} Loss {}'\n",
        "      print(template.format(epoch+1, batch_n, loss))\n",
        "\n",
        "  # saving (checkpoint) the model every 5 epochs\n",
        "  if (epoch + 1) % 5 == 0:\n",
        "    model.save_weights(checkpoint_prefix.format(epoch=epoch))\n",
        "\n",
        "  print ('Epoch {} Loss {:.4f}'.format(epoch+1, loss))\n",
        "  print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n",
        "\n",
        "model.save_weights(checkpoint_prefix.format(epoch=epoch))"
      ],
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.iter\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_1\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_2\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.decay\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.learning_rate\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-0.embeddings\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.kernel\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.bias\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.cell.kernel\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.cell.recurrent_kernel\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.cell.bias\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-0.embeddings\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.kernel\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.bias\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.cell.kernel\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.cell.recurrent_kernel\n",
            "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.cell.bias\n",
            "WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/guide/checkpoint#loading_mechanics for details.\n",
            "Epoch 1 Batch 0 Loss 4.4396467208862305\n",
            "Epoch 1 Batch 100 Loss 2.0648889541625977\n",
            "Epoch 1 Batch 200 Loss 1.7976136207580566\n",
            "Epoch 1 Batch 300 Loss 1.6339716911315918\n",
            "Epoch 1 Batch 400 Loss 1.4380426406860352\n",
            "Epoch 1 Batch 500 Loss 1.3498153686523438\n",
            "Epoch 1 Batch 600 Loss 1.3747390508651733\n",
            "Epoch 1 Loss 1.3006\n",
            "Time taken for 1 epoch 43.33114528656006 sec\n",
            "\n",
            "Epoch 2 Batch 0 Loss 1.2438489198684692\n",
            "Epoch 2 Batch 100 Loss 1.1641535758972168\n",
            "Epoch 2 Batch 200 Loss 1.1350945234298706\n",
            "Epoch 2 Batch 300 Loss 1.171319842338562\n",
            "Epoch 2 Batch 400 Loss 1.1378989219665527\n",
            "Epoch 2 Batch 500 Loss 1.205416202545166\n",
            "Epoch 2 Batch 600 Loss 1.1818456649780273\n",
            "Epoch 2 Loss 1.1555\n",
            "Time taken for 1 epoch 41.47983121871948 sec\n",
            "\n",
            "Epoch 3 Batch 0 Loss 1.0836149454116821\n",
            "Epoch 3 Batch 100 Loss 1.0164897441864014\n",
            "Epoch 3 Batch 200 Loss 1.0631693601608276\n",
            "Epoch 3 Batch 300 Loss 1.0775158405303955\n",
            "Epoch 3 Batch 400 Loss 1.0617916584014893\n",
            "Epoch 3 Batch 500 Loss 1.0771437883377075\n",
            "Epoch 3 Batch 600 Loss 1.145970344543457\n",
            "Epoch 3 Loss 1.0830\n",
            "Time taken for 1 epoch 41.84103608131409 sec\n",
            "\n",
            "Epoch 4 Batch 0 Loss 1.0599459409713745\n",
            "Epoch 4 Batch 100 Loss 0.9953482151031494\n",
            "Epoch 4 Batch 200 Loss 1.0432167053222656\n",
            "Epoch 4 Batch 300 Loss 1.038785696029663\n",
            "Epoch 4 Batch 400 Loss 1.0219039916992188\n",
            "Epoch 4 Batch 500 Loss 1.059013843536377\n",
            "Epoch 4 Batch 600 Loss 1.1013909578323364\n",
            "Epoch 4 Loss 1.0506\n",
            "Time taken for 1 epoch 41.40086078643799 sec\n",
            "\n",
            "Epoch 5 Batch 0 Loss 0.9653103351593018\n",
            "Epoch 5 Batch 100 Loss 0.9925836324691772\n",
            "Epoch 5 Batch 200 Loss 0.9803276062011719\n",
            "Epoch 5 Batch 300 Loss 0.9873319268226624\n",
            "Epoch 5 Batch 400 Loss 1.0189766883850098\n",
            "Epoch 5 Batch 500 Loss 1.0431153774261475\n",
            "Epoch 5 Batch 600 Loss 1.049531102180481\n",
            "Epoch 5 Loss 1.0088\n",
            "Time taken for 1 epoch 41.43128204345703 sec\n",
            "\n",
            "Epoch 6 Batch 0 Loss 0.9957805871963501\n",
            "Epoch 6 Batch 100 Loss 0.9342251420021057\n",
            "Epoch 6 Batch 200 Loss 0.9069965481758118\n",
            "Epoch 6 Batch 300 Loss 0.9471059441566467\n",
            "Epoch 6 Batch 400 Loss 0.9911839962005615\n",
            "Epoch 6 Batch 500 Loss 1.0060145854949951\n",
            "Epoch 6 Batch 600 Loss 1.0392155647277832\n",
            "Epoch 6 Loss 1.0492\n",
            "Time taken for 1 epoch 41.34786534309387 sec\n",
            "\n",
            "Epoch 7 Batch 0 Loss 0.9675763845443726\n",
            "Epoch 7 Batch 100 Loss 0.9186569452285767\n",
            "Epoch 7 Batch 200 Loss 0.9568895101547241\n",
            "Epoch 7 Batch 300 Loss 1.0056072473526\n",
            "Epoch 7 Batch 400 Loss 0.9782676696777344\n",
            "Epoch 7 Batch 500 Loss 1.0044050216674805\n",
            "Epoch 7 Batch 600 Loss 1.0576962232589722\n",
            "Epoch 7 Loss 0.9890\n",
            "Time taken for 1 epoch 41.4531672000885 sec\n",
            "\n",
            "Epoch 8 Batch 0 Loss 0.8962969779968262\n",
            "Epoch 8 Batch 100 Loss 0.8627636432647705\n",
            "Epoch 8 Batch 200 Loss 0.9211632013320923\n",
            "Epoch 8 Batch 300 Loss 0.9129521250724792\n",
            "Epoch 8 Batch 400 Loss 0.9570837616920471\n",
            "Epoch 8 Batch 500 Loss 0.9960259199142456\n",
            "Epoch 8 Batch 600 Loss 1.046826958656311\n",
            "Epoch 8 Loss 0.9552\n",
            "Time taken for 1 epoch 41.26309061050415 sec\n",
            "\n",
            "Epoch 9 Batch 0 Loss 0.9217801094055176\n",
            "Epoch 9 Batch 100 Loss 0.8868739604949951\n",
            "Epoch 9 Batch 200 Loss 0.918263852596283\n",
            "Epoch 9 Batch 300 Loss 0.904715359210968\n",
            "Epoch 9 Batch 400 Loss 0.8971484303474426\n",
            "Epoch 9 Batch 500 Loss 0.9179511070251465\n",
            "Epoch 9 Batch 600 Loss 0.9891240000724792\n",
            "Epoch 9 Loss 0.9965\n",
            "Time taken for 1 epoch 41.338294506073 sec\n",
            "\n",
            "Epoch 10 Batch 0 Loss 0.9234733581542969\n",
            "Epoch 10 Batch 100 Loss 0.8716515302658081\n",
            "Epoch 10 Batch 200 Loss 0.9266808032989502\n",
            "Epoch 10 Batch 300 Loss 0.9320620894432068\n",
            "Epoch 10 Batch 400 Loss 0.9038394689559937\n",
            "Epoch 10 Batch 500 Loss 1.0074396133422852\n",
            "Epoch 10 Batch 600 Loss 0.9718393087387085\n",
            "Epoch 10 Loss 0.9606\n",
            "Time taken for 1 epoch 41.30183506011963 sec\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}