wehub-resource-sync
816 lines
25 KiB
Plaintext
816 lines
25 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||
"- Author: Sebastian Raschka\n",
|
||
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "vY4SK0xKAJgm"
|
||
},
|
||
"source": [
|
||
"# Model Zoo -- Character RNN for Generating Text"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "L24QVN8uwirR"
|
||
},
|
||
"source": [
|
||
"A simple character-level RNN to generate new bits of text based on text from a novel."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"metadata": {
|
||
"colab": {},
|
||
"colab_type": "code",
|
||
"id": "moNmVfuvnImW"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Sebastian Raschka \n",
|
||
"\n",
|
||
"CPython 3.7.1\n",
|
||
"IPython 7.4.0\n",
|
||
"\n",
|
||
"torch 1.0.1.post2\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"%load_ext watermark\n",
|
||
"%watermark -a 'Sebastian Raschka' -v -p torch\n",
|
||
"\n",
|
||
"import torch\n",
|
||
"import torch.nn.functional as F\n",
|
||
"from torchtext import data\n",
|
||
"from torchtext import datasets\n",
|
||
"import time\n",
|
||
"import random\n",
|
||
"import unidecode\n",
|
||
"import string\n",
|
||
"import random\n",
|
||
"import re\n",
|
||
"\n",
|
||
"\n",
|
||
"torch.backends.cudnn.deterministic = True"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "GSRL42Qgy8I8"
|
||
},
|
||
"source": [
|
||
"## General Settings"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"metadata": {
|
||
"colab": {},
|
||
"colab_type": "code",
|
||
"id": "OvW1RgfepCBq"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"RANDOM_SEED = 123\n",
|
||
"torch.manual_seed(RANDOM_SEED)\n",
|
||
"\n",
|
||
"DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
|
||
"\n",
|
||
"TEXT_PORTION_SIZE = 200\n",
|
||
"\n",
|
||
"NUM_ITER = 20000\n",
|
||
"LEARNING_RATE = 0.005\n",
|
||
"EMBEDDING_DIM = 100\n",
|
||
"HIDDEN_DIM = 100\n",
|
||
"NUM_HIDDEN = 1"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "mQMmKUEisW4W"
|
||
},
|
||
"source": [
|
||
"## Dataset"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "4GnH64XvsV8n"
|
||
},
|
||
"source": [
|
||
"Download *[A Tale of Two Cities](http://www.gutenberg.org/files/98/98-0.txt)* by Charles Dickens from the Gutenberg Project:"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 204
|
||
},
|
||
"colab_type": "code",
|
||
"id": "WZ_4jiHVnMxN",
|
||
"outputId": "40d01460-fe20-476b-c270-1fee964bcf9f"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"--2019-04-26 04:03:36-- http://www.gutenberg.org/files/98/98-0.txt\n",
|
||
"Resolving www.gutenberg.org (www.gutenberg.org)... 152.19.134.47, 2610:28:3090:3000:0:bad:cafe:47\n",
|
||
"Connecting to www.gutenberg.org (www.gutenberg.org)|152.19.134.47|:80... connected.\n",
|
||
"HTTP request sent, awaiting response... 200 OK\n",
|
||
"Length: 804335 (785K) [text/plain]\n",
|
||
"Saving to: ‘98-0.txt.11’\n",
|
||
"\n",
|
||
"98-0.txt.11 100%[===================>] 785.48K 1.68MB/s in 0.5s \n",
|
||
"\n",
|
||
"2019-04-26 04:03:36 (1.68 MB/s) - ‘98-0.txt.11’ saved [804335/804335]\n",
|
||
"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"!wget http://www.gutenberg.org/files/98/98-0.txt"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "L-TBwKWPslPa"
|
||
},
|
||
"source": [
|
||
"Convert all characters into ASCII characters provided by `string.printable`:"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 34
|
||
},
|
||
"colab_type": "code",
|
||
"id": "i8lk0nEnih-x",
|
||
"outputId": "3c9c5cca-55a9-4e7e-a02b-64f8b9cea6c4"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~ \\t\\n\\r\\x0b\\x0c'"
|
||
]
|
||
},
|
||
"execution_count": 4,
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"string.printable"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 34
|
||
},
|
||
"colab_type": "code",
|
||
"id": "e8uNrjdtn4A8",
|
||
"outputId": "28b78ff2-8b68-4fe8-d01e-c27c8692f3ac"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Number of characters in text: 776058\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"with open('./98-0.txt', 'r') as f:\n",
|
||
" textfile = f.read()\n",
|
||
"\n",
|
||
"# convert special characters\n",
|
||
"textfile = unidecode.unidecode(textfile)\n",
|
||
"\n",
|
||
"# strip extra whitespaces\n",
|
||
"textfile = re.sub(' +',' ', textfile)\n",
|
||
"\n",
|
||
"TEXT_LENGTH = len(textfile)\n",
|
||
"\n",
|
||
"print(f'Number of characters in text: {TEXT_LENGTH}')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "JpEMNInXtZsb"
|
||
},
|
||
"source": [
|
||
"Divide the text into smaller portions:"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 68
|
||
},
|
||
"colab_type": "code",
|
||
"id": "i7JiHR1stHNF",
|
||
"outputId": "b99c6a1d-6906-4845-e9fe-20fad9ae9315"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" left his saw sticking in the firewood he was cutting, set it in\n",
|
||
"motion again; the women who had left on a door-step the little pot of\n",
|
||
"hot ashes, at which she had been trying to soften the pain in her \n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"random.seed(RANDOM_SEED)\n",
|
||
"\n",
|
||
"def random_portion(textfile):\n",
|
||
" start_index = random.randint(0, TEXT_LENGTH - TEXT_PORTION_SIZE)\n",
|
||
" end_index = start_index + TEXT_PORTION_SIZE + 1\n",
|
||
" return textfile[start_index:end_index]\n",
|
||
"\n",
|
||
"print(random_portion(textfile))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "PhZQCuBZkzgm"
|
||
},
|
||
"source": [
|
||
"Define a function to convert characters into tensors of integers (type long):\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 34
|
||
},
|
||
"colab_type": "code",
|
||
"id": "Ga-_g7Ltk479",
|
||
"outputId": "2dd6fcd9-01b4-4910-f57a-812d05b9a4d1"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"tensor([10, 11, 12, 39, 40, 41])\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"def char_to_tensor(text):\n",
|
||
" lst = [string.printable.index(c) for c in text]\n",
|
||
" tensor = torch.tensor(lst).long()\n",
|
||
" return tensor\n",
|
||
"\n",
|
||
"print(char_to_tensor('abcDEF'))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "ph8OESgtmUqD"
|
||
},
|
||
"source": [
|
||
"Putting it together to make a function that draws random batches for training:"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {},
|
||
"colab_type": "code",
|
||
"id": "U_mVm5LOl95j"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"def draw_random_sample(textfile): \n",
|
||
" text_long = char_to_tensor(random_portion(textfile))\n",
|
||
" inputs = text_long[:-1]\n",
|
||
" targets = text_long[1:]\n",
|
||
" return inputs, targets"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 425
|
||
},
|
||
"colab_type": "code",
|
||
"id": "uY8FnmSkxGZ-",
|
||
"outputId": "c332573b-5eb4-4367-bc4c-0a9a479aebb3"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"(tensor([94, 17, 18, 28, 94, 32, 18, 23, 13, 24, 32, 94, 10, 28, 94, 18, 15, 94,\n",
|
||
" 29, 17, 10, 29, 94, 32, 14, 27, 14, 94, 27, 30, 21, 14, 13, 94, 15, 24,\n",
|
||
" 27, 94, 15, 18, 16, 30, 27, 14, 28, 94, 29, 24, 24, 73, 94, 10, 23, 13,\n",
|
||
" 94, 14, 31, 14, 27, 34, 29, 17, 18, 23, 16, 96, 30, 23, 13, 14, 27, 94,\n",
|
||
" 29, 17, 14, 94, 12, 21, 24, 30, 13, 28, 94, 32, 14, 27, 14, 94, 10, 94,\n",
|
||
" 28, 30, 22, 75, 96, 96, 63, 43, 10, 21, 21, 24, 10, 62, 63, 94, 28, 10,\n",
|
||
" 18, 13, 94, 48, 27, 75, 94, 54, 29, 27, 34, 31, 14, 27, 75, 94, 63, 43,\n",
|
||
" 24, 32, 94, 13, 24, 94, 34, 24, 30, 94, 13, 24, 82, 94, 44, 94, 17, 24,\n",
|
||
" 25, 14, 94, 34, 24, 30, 94, 10, 27, 14, 94, 32, 14, 21, 21, 62, 63, 96,\n",
|
||
" 96, 44, 29, 94, 32, 10, 28, 94, 54, 29, 27, 34, 31, 14, 27, 68, 28, 94,\n",
|
||
" 16, 27, 10, 23, 13, 94, 25, 14, 12, 30, 21, 18, 10, 27, 18, 29, 34, 94,\n",
|
||
" 29, 17]),\n",
|
||
" tensor([17, 18, 28, 94, 32, 18, 23, 13, 24, 32, 94, 10, 28, 94, 18, 15, 94, 29,\n",
|
||
" 17, 10, 29, 94, 32, 14, 27, 14, 94, 27, 30, 21, 14, 13, 94, 15, 24, 27,\n",
|
||
" 94, 15, 18, 16, 30, 27, 14, 28, 94, 29, 24, 24, 73, 94, 10, 23, 13, 94,\n",
|
||
" 14, 31, 14, 27, 34, 29, 17, 18, 23, 16, 96, 30, 23, 13, 14, 27, 94, 29,\n",
|
||
" 17, 14, 94, 12, 21, 24, 30, 13, 28, 94, 32, 14, 27, 14, 94, 10, 94, 28,\n",
|
||
" 30, 22, 75, 96, 96, 63, 43, 10, 21, 21, 24, 10, 62, 63, 94, 28, 10, 18,\n",
|
||
" 13, 94, 48, 27, 75, 94, 54, 29, 27, 34, 31, 14, 27, 75, 94, 63, 43, 24,\n",
|
||
" 32, 94, 13, 24, 94, 34, 24, 30, 94, 13, 24, 82, 94, 44, 94, 17, 24, 25,\n",
|
||
" 14, 94, 34, 24, 30, 94, 10, 27, 14, 94, 32, 14, 21, 21, 62, 63, 96, 96,\n",
|
||
" 44, 29, 94, 32, 10, 28, 94, 54, 29, 27, 34, 31, 14, 27, 68, 28, 94, 16,\n",
|
||
" 27, 10, 23, 13, 94, 25, 14, 12, 30, 21, 18, 10, 27, 18, 29, 34, 94, 29,\n",
|
||
" 17, 10]))"
|
||
]
|
||
},
|
||
"execution_count": 9,
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"draw_random_sample(textfile)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "G_grdW3pxCzz"
|
||
},
|
||
"source": [
|
||
"## Model"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {},
|
||
"colab_type": "code",
|
||
"id": "nQIUm5EjxFNa"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"class RNN(torch.nn.Module):\n",
|
||
" def __init__(self, input_size, embed_size,\n",
|
||
" hidden_size, output_size, num_layers):\n",
|
||
" super(RNN, self).__init__()\n",
|
||
"\n",
|
||
" self.num_layers = num_layers\n",
|
||
" self.hidden_size = hidden_size\n",
|
||
" \n",
|
||
" self.embed = torch.nn.Embedding(input_size, hidden_size)\n",
|
||
" self.gru = torch.nn.GRU(input_size=embed_size,\n",
|
||
" hidden_size=hidden_size,\n",
|
||
" num_layers=num_layers)\n",
|
||
" self.fc = torch.nn.Linear(hidden_size, output_size)\n",
|
||
" self.init_hidden = torch.nn.Parameter(torch.zeros(\n",
|
||
" num_layers, 1, hidden_size))\n",
|
||
" \n",
|
||
" def forward(self, features, hidden):\n",
|
||
" embedded = self.embed(features.view(1, -1))\n",
|
||
" output, hidden = self.gru(embedded.view(1, 1, -1), hidden)\n",
|
||
" output = self.fc(output.view(1, -1))\n",
|
||
" return output, hidden\n",
|
||
" \n",
|
||
" def init_zero_state(self):\n",
|
||
" init_hidden = torch.zeros(self.num_layers, 1, self.hidden_size).to(DEVICE)\n",
|
||
" return init_hidden"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {},
|
||
"colab_type": "code",
|
||
"id": "Ik3NF3faxFmZ"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"torch.manual_seed(RANDOM_SEED)\n",
|
||
"model = RNN(len(string.printable), EMBEDDING_DIM, HIDDEN_DIM, len(string.printable), NUM_HIDDEN)\n",
|
||
"model = model.to(DEVICE)\n",
|
||
"optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"colab_type": "text",
|
||
"id": "Lv9Ny9di6VcI"
|
||
},
|
||
"source": [
|
||
"## Training"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {},
|
||
"colab_type": "code",
|
||
"id": "x5XlCqm3pMj2"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"def evaluate(model, prime_str='A', predict_len=100, temperature=0.8):\n",
|
||
" ## based on https://github.com/spro/practical-pytorch/\n",
|
||
" ## blob/master/char-rnn-generation/char-rnn-generation.ipynb\n",
|
||
"\n",
|
||
" hidden = model.init_zero_state()\n",
|
||
" prime_input = char_to_tensor(prime_str)\n",
|
||
" predicted = prime_str\n",
|
||
"\n",
|
||
" # Use priming string to \"build up\" hidden state\n",
|
||
" for p in range(len(prime_str) - 1):\n",
|
||
" _, hidden = model(prime_input[p].to(DEVICE), hidden.to(DEVICE))\n",
|
||
" inp = prime_input[-1]\n",
|
||
" \n",
|
||
" for p in range(predict_len):\n",
|
||
" output, hidden = model(inp.to(DEVICE), hidden.to(DEVICE))\n",
|
||
" \n",
|
||
" # Sample from the network as a multinomial distribution\n",
|
||
" output_dist = output.data.view(-1).div(temperature).exp()\n",
|
||
" top_i = torch.multinomial(output_dist, 1)[0]\n",
|
||
" \n",
|
||
" # Add predicted character to string and use as next input\n",
|
||
" predicted_char = string.printable[top_i]\n",
|
||
" predicted += predicted_char\n",
|
||
" inp = char_to_tensor(predicted_char)\n",
|
||
"\n",
|
||
" return predicted"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 13,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 3536
|
||
},
|
||
"colab_type": "code",
|
||
"id": "ZNv7_wE4prjX",
|
||
"outputId": "8ae1c36b-e4e8-4fb6-852d-ac926ad34cb8"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Time elapsed: 0.00 min\n",
|
||
"Iteration 0 | Loss 4.58\n",
|
||
"\n",
|
||
"\n",
|
||
"Th4izvh?=lw2ZaCV_}xEt5y.gA^+r\u000b",
|
||
"PgO2z@<$.1KRQe/c\\\t{a5A55Dun}_*czf.o6Hmy$l\"W@\f",
|
||
"f\u000b",
|
||
"i{7rjKsvnEMJ\n",
|
||
"mr`PaKygiE+VSbR#RF|SC^g^CZK,aenDc)t.O_\n",
|
||
"D^(M]1w'^Wd_HDws\\>_2)iavp?*c-npOvoQE>i L \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 2.63 min\n",
|
||
"Iteration 1000 | Loss 1.81\n",
|
||
"\n",
|
||
"\n",
|
||
"Th Prost\n",
|
||
"into\n",
|
||
"he forn a wock, abrould with his lother the star a caide with the Jue turnd face. Breaknay when and and of or, street were work\n",
|
||
"have the long is on the proseing bove wabres. Throk a mean h \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 5.29 min\n",
|
||
"Iteration 2000 | Loss 1.72\n",
|
||
"\n",
|
||
"\n",
|
||
"Ther face. And civery ire head shook the lange's was note my booked she cray. The grance for that the with Lerary swere were, and for young to-the\n",
|
||
"wank the tanger brother whas at a for the requestone-st \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 7.91 min\n",
|
||
"Iteration 3000 | Loss 1.73\n",
|
||
"\n",
|
||
"\n",
|
||
"Thou my menal known a purntatieful a might\n",
|
||
"Frent fargefuch by sour that reforned after as as a mists\n",
|
||
"and the countice of the Founk\n",
|
||
"\n",
|
||
"\n",
|
||
"\n",
|
||
"\"I among him your for the you glason in?\"\n",
|
||
"\n",
|
||
"\"I constrance yhabuing a \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 10.55 min\n",
|
||
"Iteration 4000 | Loss 1.77\n",
|
||
"\n",
|
||
"\n",
|
||
"The seeantelition pricomer; I have\n",
|
||
"had the passess bestious had be patriender one up thow, such the even the line\n",
|
||
"and that ins show was somen of his openey, but fine had a raghter?\n",
|
||
"\n",
|
||
"\"I! And at a sifulra \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 13.17 min\n",
|
||
"Iteration 5000 | Loss 1.46\n",
|
||
"\n",
|
||
"\n",
|
||
"The Bask tree. \"The intame!\"\n",
|
||
"\n",
|
||
"\"Neothing and fam and if you brow lisert, to the mouther desk to an to the Gells that immered of the\n",
|
||
"indence an aftionation bud, undering to went remark down off work; doe! \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 15.83 min\n",
|
||
"Iteration 6000 | Loss 1.64\n",
|
||
"\n",
|
||
"\n",
|
||
"The Pross. What\n",
|
||
"of moon, and worth her knitting, and is he see myself the was seeper on prisoner her been on him our, and\n",
|
||
"yet in the poors; is stooness of a morned this things more, were benthell name, \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 18.47 min\n",
|
||
"Iteration 7000 | Loss 1.64\n",
|
||
"\n",
|
||
"\n",
|
||
"The here an the ferty care it was\n",
|
||
"of the streach. As Miss Pross Borring of her surfounds of comprelegual\n",
|
||
"saken which his returnes, shall in Heaved the arrows\n",
|
||
"of the retore, then for Defarge. Jark, he wa \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 21.12 min\n",
|
||
"Iteration 8000 | Loss 1.66\n",
|
||
"\n",
|
||
"\n",
|
||
"Thur and the decients than any.\n",
|
||
"\n",
|
||
"Monsiever such put her cite out over the cermarded and in herce then the repariey who\n",
|
||
"grance the stalled be of the own and conversicted way of his anterom\n",
|
||
"cold the cirse \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 23.77 min\n",
|
||
"Iteration 9000 | Loss 1.53\n",
|
||
"\n",
|
||
"\n",
|
||
"Thrat to his man that extenss of the said her and had and world at it, she had was\n",
|
||
"as breat--how had asseet triatile of the pationed, and\n",
|
||
"that worked he works of one and nobainly, and out of that at the \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 26.43 min\n",
|
||
"Iteration 10000 | Loss 1.30\n",
|
||
"\n",
|
||
"\n",
|
||
"Ther his moth wooten a new blood, a sile, the lactriden\n",
|
||
"nother were noter, who had from his father to prettorers his\n",
|
||
"fation. Then. He are is him a sloke it soits in him woired to the paper women, maning \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 29.07 min\n",
|
||
"Iteration 11000 | Loss 1.73\n",
|
||
"\n",
|
||
"\n",
|
||
"The eighs while Miss Pross was\n",
|
||
"saying a could they last the done by, and pressed to\n",
|
||
"the been hackeful hight in mending, and to the done into-raid to\n",
|
||
"have little faming shall now, with the said to go of \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 31.72 min\n",
|
||
"Iteration 12000 | Loss 1.68\n",
|
||
"\n",
|
||
"\n",
|
||
"The here. It were would done.\n",
|
||
"\n",
|
||
"\"It alread, I was say?\" seen in not in the culles the sunded Miss, sure to be there were the would close\n",
|
||
"he dark see radfe taken it is instend me had done-all I spy so\n",
|
||
"str \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 34.35 min\n",
|
||
"Iteration 13000 | Loss 1.42\n",
|
||
"\n",
|
||
"\n",
|
||
"The part, but, at had were tosen in\n",
|
||
"it are of a proined serverently passing the fars, and the\n",
|
||
"friended that a fiffer the knouttial backle and day, list, and from to could my\n",
|
||
"deting; and very dark of the \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 37.00 min\n",
|
||
"Iteration 14000 | Loss 1.78\n",
|
||
"\n",
|
||
"\n",
|
||
"Thre it was a days.\n",
|
||
"\n",
|
||
"\"You and deperianned of the moved there way, and a socions of the proppiouse and this must a dively?\n",
|
||
"\n",
|
||
"\"Yest!\" (And in the care befon this there,\" asked nother, to the two in this ex \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 39.63 min\n",
|
||
"Iteration 15000 | Loss 1.54\n",
|
||
"\n",
|
||
"\n",
|
||
"Tho a man in all looking\n",
|
||
"the mannen were trangs; he more at man and in the had believe the sick of their an\n",
|
||
"than the man the prioned in a golderate scattered no stup, and look, all thoused shall\n",
|
||
"law sca \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 42.28 min\n",
|
||
"Iteration 16000 | Loss 1.52\n",
|
||
"\n",
|
||
"\n",
|
||
"Thrishe forth, his have like him\n",
|
||
"and words of it is a peeched in the eyes farge what\n",
|
||
"it went exciect the deing and the mittions.\n",
|
||
"\n",
|
||
"The mounged the repalling's citines of mineurmt you not thinks,\n",
|
||
" Charlee \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 44.93 min\n",
|
||
"Iteration 17000 | Loss 1.58\n",
|
||
"\n",
|
||
"\n",
|
||
"Thrithest the prisonened I\n",
|
||
"staid be, short, and not morright door with with the mitting to my worthud no paid\n",
|
||
"it.\"\n",
|
||
"\n",
|
||
"\"He do I as a more through a passed and go more.\n",
|
||
"\n",
|
||
"No and me, the far bold to fears and \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 47.56 min\n",
|
||
"Iteration 18000 | Loss 1.49\n",
|
||
"\n",
|
||
"\n",
|
||
"Tho\n",
|
||
"you would fro in his intides rather sation and chocal went in the things, asked the have hand of the\n",
|
||
"distened did of the cately roar chifulures. What the His of a not his have pourty\n",
|
||
"the took this l \n",
|
||
"\n",
|
||
"==================================================\n",
|
||
"Time elapsed: 50.19 min\n",
|
||
"Iteration 19000 | Loss 1.78\n",
|
||
"\n",
|
||
"\n",
|
||
"Thragges,\" said some of\n",
|
||
"a puncher in the Gabody old, was a Fants tall to know of the complight--seat\n",
|
||
"more inten asse interancame my any went med Courable hands in that he behing make no will never see t \n",
|
||
"\n",
|
||
"==================================================\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"start_time = time.time()\n",
|
||
"for iteration in range(NUM_ITER):\n",
|
||
"\n",
|
||
" \n",
|
||
" ### FORWARD AND BACK PROP\n",
|
||
"\n",
|
||
" hidden = model.init_zero_state()\n",
|
||
" optimizer.zero_grad()\n",
|
||
" \n",
|
||
" loss = 0.\n",
|
||
" inputs, targets = draw_random_sample(textfile)\n",
|
||
" inputs, targets = inputs.to(DEVICE), targets.to(DEVICE)\n",
|
||
" for c in range(TEXT_PORTION_SIZE):\n",
|
||
" outputs, hidden = model(inputs[c], hidden)\n",
|
||
" loss += F.cross_entropy(outputs, targets[c].view(1))\n",
|
||
"\n",
|
||
" loss /= TEXT_PORTION_SIZE\n",
|
||
" loss.backward()\n",
|
||
" \n",
|
||
" ### UPDATE MODEL PARAMETERS\n",
|
||
" optimizer.step()\n",
|
||
"\n",
|
||
" ### LOGGING\n",
|
||
" with torch.set_grad_enabled(False):\n",
|
||
" if iteration % 1000 == 0:\n",
|
||
" print(f'Time elapsed: {(time.time() - start_time)/60:.2f} min')\n",
|
||
" print(f'Iteration {iteration} | Loss {loss.item():.2f}\\n\\n')\n",
|
||
" print(evaluate(model, 'Th', 200), '\\n')\n",
|
||
" print(50*'=')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {},
|
||
"colab_type": "code",
|
||
"id": "mEvBe4V0ykSl"
|
||
},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"accelerator": "GPU",
|
||
"colab": {
|
||
"collapsed_sections": [],
|
||
"name": "char-rnn.ipynb",
|
||
"provenance": [],
|
||
"version": "0.3.2"
|
||
},
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.7.1"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 2
|
||
}
|