wehub-resource-sync
748 lines
31 KiB
Plaintext
748 lines
31 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||
"- Author: Sebastian Raschka\n",
|
||
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Sebastian Raschka \n",
|
||
"\n",
|
||
"CPython 3.6.1\n",
|
||
"IPython 6.0.0\n",
|
||
"\n",
|
||
"tensorflow 1.2.0\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"%load_ext watermark\n",
|
||
"%watermark -a 'Sebastian Raschka' -v -p tensorflow"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Model Zoo -- Convolutional Neural Network (VGG16)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"The VGG-16 Convolutional Neural Network Architecture [1] implemented in TensorFlow and trained on Cifar-10 [2, 3] images.\n",
|
||
"\n",
|
||
"References:\n",
|
||
"\n",
|
||
"- [1] Simonyan, K., & Zisserman, A. (2015). [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556). International Conference on Learning Representations (ICRL), 1–14. https://doi.org/10.1016/j.infsof.2008.09.005\n",
|
||
"- [2] Krizhevsky, A. (2009). [Learning Multiple Layers of Features from Tiny Images](https://doi.org/10.1.1.222.9220 http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.222.9220&rep=rep1&type=pdf). Science Department, University of Toronto.\n",
|
||
"- [3] https://www.cs.toronto.edu/~kriz/cifar.html"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"ename": "TypeError",
|
||
"evalue": "insert() takes exactly 2 arguments (1 given)",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
|
||
"\u001b[0;32m<ipython-input-1-0beb05071f51>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'..'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mhelper\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdownload_and_extract_cifar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||
"\u001b[0;31mTypeError\u001b[0m: insert() takes exactly 2 arguments (1 given)"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"##########################\n",
|
||
"### DATASET\n",
|
||
"##########################\n",
|
||
"\n",
|
||
"# load utilities from ../helper.py\n",
|
||
"import sys\n",
|
||
"sys.path.insert(0, '..') \n",
|
||
"from helper import download_and_extract_cifar\n",
|
||
"from helper import Cifar10Loader\n",
|
||
"\n",
|
||
"dest = download_and_extract_cifar('./cifar-10')\n",
|
||
"cifar = Cifar10Loader(dest, normalize=True, \n",
|
||
" zero_center=True,\n",
|
||
" channel_mean_center=False)\n",
|
||
"cifar.num_train\n",
|
||
"\n",
|
||
"X, y = cifar.load_test()\n",
|
||
"half = cifar.num_test // 2\n",
|
||
"X_test, X_valid = X[:half], X[half:]\n",
|
||
"y_test, y_valid = y[:half], y[half:]\n",
|
||
"\n",
|
||
"del X, y"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"'/gpu:0'"
|
||
]
|
||
},
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import tensorflow as tf\n",
|
||
"import numpy as np\n",
|
||
"\n",
|
||
"tf.test.gpu_device_name()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Minibatch: 0200 | Cost: 9.004\n",
|
||
"Minibatch: 0400 | Cost: 3.877\n",
|
||
"Minibatch: 0600 | Cost: 3.213\n",
|
||
"Minibatch: 0800 | Cost: 2.410\n",
|
||
"Minibatch: 1000 | Cost: 2.384\n",
|
||
"Minibatch: 1200 | Cost: 2.250\n",
|
||
"Minibatch: 1400 | Cost: 2.352\n",
|
||
"Epoch: 001 | AvgCost: 1897.994 | Train/Valid ACC: 0.152/0.145\n",
|
||
"Minibatch: 0200 | Cost: 2.433\n",
|
||
"Minibatch: 0400 | Cost: 2.426\n",
|
||
"Minibatch: 0600 | Cost: 2.202\n",
|
||
"Minibatch: 0800 | Cost: 2.221\n",
|
||
"Minibatch: 1000 | Cost: 2.112\n",
|
||
"Minibatch: 1200 | Cost: 2.182\n",
|
||
"Minibatch: 1400 | Cost: 2.382\n",
|
||
"Epoch: 002 | AvgCost: 2.219 | Train/Valid ACC: 0.176/0.168\n",
|
||
"Minibatch: 0200 | Cost: 2.145\n",
|
||
"Minibatch: 0400 | Cost: 2.089\n",
|
||
"Minibatch: 0600 | Cost: 2.221\n",
|
||
"Minibatch: 0800 | Cost: 1.903\n",
|
||
"Minibatch: 1000 | Cost: 2.296\n",
|
||
"Minibatch: 1200 | Cost: 2.169\n",
|
||
"Minibatch: 1400 | Cost: 2.164\n",
|
||
"Epoch: 003 | AvgCost: 2.168 | Train/Valid ACC: 0.166/0.173\n",
|
||
"Minibatch: 0200 | Cost: 2.094\n",
|
||
"Minibatch: 0400 | Cost: 2.015\n",
|
||
"Minibatch: 0600 | Cost: 2.217\n",
|
||
"Minibatch: 0800 | Cost: 2.028\n",
|
||
"Minibatch: 1000 | Cost: 2.265\n",
|
||
"Minibatch: 1200 | Cost: 2.103\n",
|
||
"Minibatch: 1400 | Cost: 2.060\n",
|
||
"Epoch: 004 | AvgCost: 2.144 | Train/Valid ACC: 0.189/0.188\n",
|
||
"Minibatch: 0200 | Cost: 2.310\n",
|
||
"Minibatch: 0400 | Cost: 1.986\n",
|
||
"Minibatch: 0600 | Cost: 1.844\n",
|
||
"Minibatch: 0800 | Cost: 2.183\n",
|
||
"Minibatch: 1000 | Cost: 2.040\n",
|
||
"Minibatch: 1200 | Cost: 2.004\n",
|
||
"Minibatch: 1400 | Cost: 1.987\n",
|
||
"Epoch: 005 | AvgCost: 2.075 | Train/Valid ACC: 0.180/0.181\n",
|
||
"Minibatch: 0200 | Cost: 1.849\n",
|
||
"Minibatch: 0400 | Cost: 1.898\n",
|
||
"Minibatch: 0600 | Cost: 1.996\n",
|
||
"Minibatch: 0800 | Cost: 1.968\n",
|
||
"Minibatch: 1000 | Cost: 1.987\n",
|
||
"Minibatch: 1200 | Cost: 2.003\n",
|
||
"Minibatch: 1400 | Cost: 2.041\n",
|
||
"Epoch: 006 | AvgCost: 2.010 | Train/Valid ACC: 0.176/0.176\n",
|
||
"Minibatch: 0200 | Cost: 2.419\n",
|
||
"Minibatch: 0400 | Cost: 2.109\n",
|
||
"Minibatch: 0600 | Cost: 1.902\n",
|
||
"Minibatch: 0800 | Cost: 1.937\n",
|
||
"Minibatch: 1000 | Cost: 1.813\n",
|
||
"Minibatch: 1200 | Cost: 1.990\n",
|
||
"Minibatch: 1400 | Cost: 1.662\n",
|
||
"Epoch: 007 | AvgCost: 1.913 | Train/Valid ACC: 0.256/0.251\n",
|
||
"Minibatch: 0200 | Cost: 2.036\n",
|
||
"Minibatch: 0400 | Cost: 1.690\n",
|
||
"Minibatch: 0600 | Cost: 1.866\n",
|
||
"Minibatch: 0800 | Cost: 1.574\n",
|
||
"Minibatch: 1000 | Cost: 1.886\n",
|
||
"Minibatch: 1200 | Cost: 1.720\n",
|
||
"Minibatch: 1400 | Cost: 1.937\n",
|
||
"Epoch: 008 | AvgCost: 1.769 | Train/Valid ACC: 0.270/0.267\n",
|
||
"Minibatch: 0200 | Cost: 1.569\n",
|
||
"Minibatch: 0400 | Cost: 1.800\n",
|
||
"Minibatch: 0600 | Cost: 1.541\n",
|
||
"Minibatch: 0800 | Cost: 1.499\n",
|
||
"Minibatch: 1000 | Cost: 1.596\n",
|
||
"Minibatch: 1200 | Cost: 1.994\n",
|
||
"Minibatch: 1400 | Cost: 1.283\n",
|
||
"Epoch: 009 | AvgCost: 1.653 | Train/Valid ACC: 0.343/0.337\n",
|
||
"Minibatch: 0200 | Cost: 1.415\n",
|
||
"Minibatch: 0400 | Cost: 1.648\n",
|
||
"Minibatch: 0600 | Cost: 1.487\n",
|
||
"Minibatch: 0800 | Cost: 1.454\n",
|
||
"Minibatch: 1000 | Cost: 1.266\n",
|
||
"Minibatch: 1200 | Cost: 1.267\n",
|
||
"Minibatch: 1400 | Cost: 1.589\n",
|
||
"Epoch: 010 | AvgCost: 1.526 | Train/Valid ACC: 0.469/0.459\n",
|
||
"Minibatch: 0200 | Cost: 1.585\n",
|
||
"Minibatch: 0400 | Cost: 1.449\n",
|
||
"Minibatch: 0600 | Cost: 1.340\n",
|
||
"Minibatch: 0800 | Cost: 1.297\n",
|
||
"Minibatch: 1000 | Cost: 1.767\n",
|
||
"Minibatch: 1200 | Cost: 1.088\n",
|
||
"Minibatch: 1400 | Cost: 1.159\n",
|
||
"Epoch: 011 | AvgCost: 1.342 | Train/Valid ACC: 0.536/0.517\n",
|
||
"Minibatch: 0200 | Cost: 1.408\n",
|
||
"Minibatch: 0400 | Cost: 1.510\n",
|
||
"Minibatch: 0600 | Cost: 1.424\n",
|
||
"Minibatch: 0800 | Cost: 1.019\n",
|
||
"Minibatch: 1000 | Cost: 1.069\n",
|
||
"Minibatch: 1200 | Cost: 1.465\n",
|
||
"Minibatch: 1400 | Cost: 1.289\n",
|
||
"Epoch: 012 | AvgCost: 1.203 | Train/Valid ACC: 0.590/0.576\n",
|
||
"Minibatch: 0200 | Cost: 0.670\n",
|
||
"Minibatch: 0400 | Cost: 0.993\n",
|
||
"Minibatch: 0600 | Cost: 0.998\n",
|
||
"Minibatch: 0800 | Cost: 1.103\n",
|
||
"Minibatch: 1000 | Cost: 1.127\n",
|
||
"Minibatch: 1200 | Cost: 0.969\n",
|
||
"Minibatch: 1400 | Cost: 1.216\n",
|
||
"Epoch: 013 | AvgCost: 1.111 | Train/Valid ACC: 0.634/0.603\n",
|
||
"Minibatch: 0200 | Cost: 1.262\n",
|
||
"Minibatch: 0400 | Cost: 1.147\n",
|
||
"Minibatch: 0600 | Cost: 0.660\n",
|
||
"Minibatch: 0800 | Cost: 0.970\n",
|
||
"Minibatch: 1000 | Cost: 0.716\n",
|
||
"Minibatch: 1200 | Cost: 1.017\n",
|
||
"Minibatch: 1400 | Cost: 1.264\n",
|
||
"Epoch: 014 | AvgCost: 1.036 | Train/Valid ACC: 0.638/0.617\n",
|
||
"Minibatch: 0200 | Cost: 0.846\n",
|
||
"Minibatch: 0400 | Cost: 0.843\n",
|
||
"Minibatch: 0600 | Cost: 0.792\n",
|
||
"Minibatch: 0800 | Cost: 0.718\n",
|
||
"Minibatch: 1000 | Cost: 1.141\n",
|
||
"Minibatch: 1200 | Cost: 1.024\n",
|
||
"Minibatch: 1400 | Cost: 0.877\n",
|
||
"Epoch: 015 | AvgCost: 0.949 | Train/Valid ACC: 0.683/0.654\n",
|
||
"Minibatch: 0200 | Cost: 0.463\n",
|
||
"Minibatch: 0400 | Cost: 0.854\n",
|
||
"Minibatch: 0600 | Cost: 0.820\n",
|
||
"Minibatch: 0800 | Cost: 0.952\n",
|
||
"Minibatch: 1000 | Cost: 0.819\n",
|
||
"Minibatch: 1200 | Cost: 0.648\n",
|
||
"Minibatch: 1400 | Cost: 1.030\n",
|
||
"Epoch: 016 | AvgCost: 0.872 | Train/Valid ACC: 0.738/0.697\n",
|
||
"Minibatch: 0200 | Cost: 0.862\n",
|
||
"Minibatch: 0400 | Cost: 1.010\n",
|
||
"Minibatch: 0600 | Cost: 0.991\n",
|
||
"Minibatch: 0800 | Cost: 0.664\n",
|
||
"Minibatch: 1000 | Cost: 0.604\n",
|
||
"Minibatch: 1200 | Cost: 0.879\n",
|
||
"Minibatch: 1400 | Cost: 0.791\n",
|
||
"Epoch: 017 | AvgCost: 0.823 | Train/Valid ACC: 0.753/0.698\n",
|
||
"Minibatch: 0200 | Cost: 0.823\n",
|
||
"Minibatch: 0400 | Cost: 0.619\n",
|
||
"Minibatch: 0600 | Cost: 0.764\n",
|
||
"Minibatch: 0800 | Cost: 1.127\n",
|
||
"Minibatch: 1000 | Cost: 0.623\n",
|
||
"Minibatch: 1200 | Cost: 0.796\n",
|
||
"Minibatch: 1400 | Cost: 0.694\n",
|
||
"Epoch: 018 | AvgCost: 0.771 | Train/Valid ACC: 0.769/0.709\n",
|
||
"Minibatch: 0200 | Cost: 0.431\n",
|
||
"Minibatch: 0400 | Cost: 0.688\n",
|
||
"Minibatch: 0600 | Cost: 0.406\n",
|
||
"Minibatch: 0800 | Cost: 0.906\n",
|
||
"Minibatch: 1000 | Cost: 0.607\n",
|
||
"Minibatch: 1200 | Cost: 0.445\n",
|
||
"Minibatch: 1400 | Cost: 0.660\n",
|
||
"Epoch: 019 | AvgCost: 0.721 | Train/Valid ACC: 0.780/0.707\n",
|
||
"Minibatch: 0200 | Cost: 0.740\n",
|
||
"Minibatch: 0400 | Cost: 0.542\n",
|
||
"Minibatch: 0600 | Cost: 0.435\n",
|
||
"Minibatch: 0800 | Cost: 0.953\n",
|
||
"Minibatch: 1000 | Cost: 0.954\n",
|
||
"Minibatch: 1200 | Cost: 0.544\n",
|
||
"Minibatch: 1400 | Cost: 0.855\n",
|
||
"Epoch: 020 | AvgCost: 0.672 | Train/Valid ACC: 0.779/0.717\n",
|
||
"Minibatch: 0200 | Cost: 0.432\n",
|
||
"Minibatch: 0400 | Cost: 0.376\n",
|
||
"Minibatch: 0600 | Cost: 0.477\n",
|
||
"Minibatch: 0800 | Cost: 0.476\n",
|
||
"Minibatch: 1000 | Cost: 0.950\n",
|
||
"Minibatch: 1200 | Cost: 0.468\n",
|
||
"Minibatch: 1400 | Cost: 0.595\n",
|
||
"Epoch: 021 | AvgCost: 0.636 | Train/Valid ACC: 0.818/0.737\n",
|
||
"Minibatch: 0200 | Cost: 1.114\n",
|
||
"Minibatch: 0400 | Cost: 0.519\n",
|
||
"Minibatch: 0600 | Cost: 0.623\n",
|
||
"Minibatch: 0800 | Cost: 0.721\n",
|
||
"Minibatch: 1000 | Cost: 0.880\n",
|
||
"Minibatch: 1200 | Cost: 0.654\n",
|
||
"Minibatch: 1400 | Cost: 0.378\n",
|
||
"Epoch: 022 | AvgCost: 0.596 | Train/Valid ACC: 0.825/0.742\n",
|
||
"Minibatch: 0200 | Cost: 0.541\n",
|
||
"Minibatch: 0400 | Cost: 0.478\n",
|
||
"Minibatch: 0600 | Cost: 0.642\n",
|
||
"Minibatch: 0800 | Cost: 0.871\n",
|
||
"Minibatch: 1000 | Cost: 0.710\n",
|
||
"Minibatch: 1200 | Cost: 0.682\n",
|
||
"Minibatch: 1400 | Cost: 0.671\n",
|
||
"Epoch: 023 | AvgCost: 0.663 | Train/Valid ACC: 0.776/0.712\n",
|
||
"Minibatch: 0200 | Cost: 0.498\n",
|
||
"Minibatch: 0400 | Cost: 0.379\n",
|
||
"Minibatch: 0600 | Cost: 0.535\n",
|
||
"Minibatch: 0800 | Cost: 0.435\n",
|
||
"Minibatch: 1000 | Cost: 0.562\n",
|
||
"Minibatch: 1200 | Cost: 0.777\n",
|
||
"Minibatch: 1400 | Cost: 0.635\n",
|
||
"Epoch: 024 | AvgCost: 0.539 | Train/Valid ACC: 0.853/0.759\n",
|
||
"Minibatch: 0200 | Cost: 0.331\n",
|
||
"Minibatch: 0400 | Cost: 0.419\n",
|
||
"Minibatch: 0600 | Cost: 0.497\n",
|
||
"Minibatch: 0800 | Cost: 0.619\n",
|
||
"Minibatch: 1000 | Cost: 0.620\n",
|
||
"Minibatch: 1200 | Cost: 0.196\n",
|
||
"Minibatch: 1400 | Cost: 0.648\n",
|
||
"Epoch: 025 | AvgCost: 0.498 | Train/Valid ACC: 0.849/0.738\n",
|
||
"Minibatch: 0200 | Cost: 0.247\n",
|
||
"Minibatch: 0400 | Cost: 0.263\n",
|
||
"Minibatch: 0600 | Cost: 0.183\n",
|
||
"Minibatch: 0800 | Cost: 0.251\n",
|
||
"Minibatch: 1000 | Cost: 0.251\n",
|
||
"Minibatch: 1200 | Cost: 0.447\n",
|
||
"Minibatch: 1400 | Cost: 0.443\n",
|
||
"Epoch: 026 | AvgCost: 0.469 | Train/Valid ACC: 0.850/0.744\n",
|
||
"Minibatch: 0200 | Cost: 0.390\n",
|
||
"Minibatch: 0400 | Cost: 0.325\n",
|
||
"Minibatch: 0600 | Cost: 0.404\n",
|
||
"Minibatch: 0800 | Cost: 0.363\n",
|
||
"Minibatch: 1000 | Cost: 0.655\n",
|
||
"Minibatch: 1200 | Cost: 0.195\n",
|
||
"Minibatch: 1400 | Cost: 0.647\n",
|
||
"Epoch: 027 | AvgCost: 0.485 | Train/Valid ACC: 0.855/0.751\n",
|
||
"Minibatch: 0200 | Cost: 0.580\n",
|
||
"Minibatch: 0400 | Cost: 0.177\n",
|
||
"Minibatch: 0600 | Cost: 0.317\n",
|
||
"Minibatch: 0800 | Cost: 0.249\n",
|
||
"Minibatch: 1000 | Cost: 0.287\n",
|
||
"Minibatch: 1200 | Cost: 0.282\n",
|
||
"Minibatch: 1400 | Cost: 0.429\n",
|
||
"Epoch: 028 | AvgCost: 0.416 | Train/Valid ACC: 0.887/0.759\n",
|
||
"Minibatch: 0200 | Cost: 0.233\n",
|
||
"Minibatch: 0400 | Cost: 0.118\n",
|
||
"Minibatch: 0600 | Cost: 0.218\n",
|
||
"Minibatch: 0800 | Cost: 0.195\n",
|
||
"Minibatch: 1000 | Cost: 0.208\n",
|
||
"Minibatch: 1200 | Cost: 0.188\n",
|
||
"Minibatch: 1400 | Cost: 0.509\n",
|
||
"Epoch: 029 | AvgCost: 0.488 | Train/Valid ACC: 0.874/0.767\n",
|
||
"Minibatch: 0200 | Cost: 0.313\n",
|
||
"Minibatch: 0400 | Cost: 0.508\n",
|
||
"Minibatch: 0600 | Cost: 0.553\n",
|
||
"Minibatch: 0800 | Cost: 0.385\n",
|
||
"Minibatch: 1000 | Cost: 0.272\n",
|
||
"Minibatch: 1200 | Cost: 0.366\n",
|
||
"Minibatch: 1400 | Cost: 0.156\n",
|
||
"Epoch: 030 | AvgCost: 0.404 | Train/Valid ACC: 0.909/0.776\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"##########################\n",
|
||
"### SETTINGS\n",
|
||
"##########################\n",
|
||
"\n",
|
||
"# Hyperparameters\n",
|
||
"learning_rate = 0.001\n",
|
||
"training_epochs = 30\n",
|
||
"batch_size = 32\n",
|
||
"\n",
|
||
"# Other\n",
|
||
"print_interval = 200\n",
|
||
"\n",
|
||
"# Architecture\n",
|
||
"image_width, image_height, image_depth = 32, 32, 3\n",
|
||
"n_classes = 10\n",
|
||
"\n",
|
||
"\n",
|
||
"##########################\n",
|
||
"### WRAPPER FUNCTIONS\n",
|
||
"##########################\n",
|
||
"\n",
|
||
"def conv_layer(input, input_channels, output_channels, \n",
|
||
" kernel_size, strides, scope, padding='SAME'):\n",
|
||
" with tf.name_scope(scope):\n",
|
||
" weights_shape = kernel_size + [input_channels, output_channels]\n",
|
||
" weights = tf.Variable(tf.truncated_normal(shape=weights_shape,\n",
|
||
" mean=0.0,\n",
|
||
" stddev=0.1,\n",
|
||
" dtype=tf.float32),\n",
|
||
" name='weights')\n",
|
||
" biases = tf.Variable(tf.zeros(shape=[output_channels]),\n",
|
||
" name='biases')\n",
|
||
" conv = tf.nn.conv2d(input=input,\n",
|
||
" filter=weights,\n",
|
||
" strides=strides,\n",
|
||
" padding=padding,\n",
|
||
" name='convolution')\n",
|
||
" out = tf.nn.bias_add(conv, biases, name='logits')\n",
|
||
" out = tf.nn.relu(out, name='activation')\n",
|
||
" return out\n",
|
||
"\n",
|
||
"\n",
|
||
"def fc_layer(input, output_nodes, scope,\n",
|
||
" activation=None, seed=None):\n",
|
||
" with tf.name_scope(scope):\n",
|
||
" shape = int(np.prod(input.get_shape()[1:]))\n",
|
||
" flat_input = tf.reshape(input, [-1, shape])\n",
|
||
" weights = tf.Variable(tf.truncated_normal(shape=[shape,\n",
|
||
" output_nodes],\n",
|
||
" mean=0.0,\n",
|
||
" stddev=0.1,\n",
|
||
" dtype=tf.float32,\n",
|
||
" seed=seed),\n",
|
||
" name='weights')\n",
|
||
" biases = tf.Variable(tf.zeros(shape=[output_nodes]),\n",
|
||
" name='biases')\n",
|
||
" act = tf.nn.bias_add(tf.matmul(flat_input, weights), biases, \n",
|
||
" name='logits')\n",
|
||
"\n",
|
||
" if activation is not None:\n",
|
||
" act = activation(act, name='activation')\n",
|
||
"\n",
|
||
" return act\n",
|
||
"\n",
|
||
"\n",
|
||
"##########################\n",
|
||
"### GRAPH DEFINITION\n",
|
||
"##########################\n",
|
||
"\n",
|
||
"g = tf.Graph()\n",
|
||
"with g.as_default():\n",
|
||
"\n",
|
||
" # Input data\n",
|
||
" tf_x = tf.placeholder(tf.float32, [None, image_width, image_height, image_depth], name='features')\n",
|
||
" tf_y = tf.placeholder(tf.float32, [None, n_classes], name='targets')\n",
|
||
" \n",
|
||
" ##########################\n",
|
||
" ### VGG16 Model\n",
|
||
" ##########################\n",
|
||
"\n",
|
||
" # =========\n",
|
||
" # BLOCK 1\n",
|
||
" # =========\n",
|
||
" conv_layer_1 = conv_layer(input=tf_x,\n",
|
||
" input_channels=3,\n",
|
||
" output_channels=64,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv1')\n",
|
||
" \n",
|
||
" conv_layer_2 = conv_layer(input=conv_layer_1,\n",
|
||
" input_channels=64,\n",
|
||
" output_channels=64,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv2') \n",
|
||
" \n",
|
||
" pool_layer_1 = tf.nn.max_pool(conv_layer_2,\n",
|
||
" ksize=[1, 2, 2, 1], \n",
|
||
" strides=[1, 2, 2, 1],\n",
|
||
" padding='SAME',\n",
|
||
" name='pool1') \n",
|
||
" # =========\n",
|
||
" # BLOCK 2\n",
|
||
" # =========\n",
|
||
" conv_layer_3 = conv_layer(input=pool_layer_1,\n",
|
||
" input_channels=64,\n",
|
||
" output_channels=128,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv3') \n",
|
||
" \n",
|
||
" conv_layer_4 = conv_layer(input=conv_layer_3,\n",
|
||
" input_channels=128,\n",
|
||
" output_channels=128,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv4') \n",
|
||
" \n",
|
||
" pool_layer_2 = tf.nn.max_pool(conv_layer_4,\n",
|
||
" ksize=[1, 2, 2, 1], \n",
|
||
" strides=[1, 2, 2, 1],\n",
|
||
" padding='SAME',\n",
|
||
" name='pool2') \n",
|
||
" # =========\n",
|
||
" # BLOCK 3\n",
|
||
" # =========\n",
|
||
" conv_layer_5 = conv_layer(input=pool_layer_2,\n",
|
||
" input_channels=128,\n",
|
||
" output_channels=256,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv5') \n",
|
||
" \n",
|
||
" conv_layer_6 = conv_layer(input=conv_layer_5,\n",
|
||
" input_channels=256,\n",
|
||
" output_channels=256,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv6') \n",
|
||
" \n",
|
||
" conv_layer_7 = conv_layer(input=conv_layer_6,\n",
|
||
" input_channels=256,\n",
|
||
" output_channels=256,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv7')\n",
|
||
" \n",
|
||
" pool_layer_3 = tf.nn.max_pool(conv_layer_7,\n",
|
||
" ksize=[1, 2, 2, 1], \n",
|
||
" strides=[1, 2, 2, 1],\n",
|
||
" padding='SAME',\n",
|
||
" name='pool3') \n",
|
||
" # =========\n",
|
||
" # BLOCK 4\n",
|
||
" # =========\n",
|
||
" conv_layer_8 = conv_layer(input=pool_layer_3,\n",
|
||
" input_channels=256,\n",
|
||
" output_channels=512,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv8') \n",
|
||
" \n",
|
||
" conv_layer_9 = conv_layer(input=conv_layer_8,\n",
|
||
" input_channels=512,\n",
|
||
" output_channels=512,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv9') \n",
|
||
" \n",
|
||
" conv_layer_10 = conv_layer(input=conv_layer_9,\n",
|
||
" input_channels=512,\n",
|
||
" output_channels=512,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv10') \n",
|
||
" \n",
|
||
" pool_layer_4 = tf.nn.max_pool(conv_layer_10,\n",
|
||
" ksize=[1, 2, 2, 1], \n",
|
||
" strides=[1, 2, 2, 1],\n",
|
||
" padding='SAME',\n",
|
||
" name='pool4') \n",
|
||
" # =========\n",
|
||
" # BLOCK 5\n",
|
||
" # =========\n",
|
||
" conv_layer_11 = conv_layer(input=pool_layer_4,\n",
|
||
" input_channels=512,\n",
|
||
" output_channels=512,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv11') \n",
|
||
" \n",
|
||
" conv_layer_12 = conv_layer(input=conv_layer_11,\n",
|
||
" input_channels=512,\n",
|
||
" output_channels=512,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv12') \n",
|
||
"\n",
|
||
" conv_layer_13 = conv_layer(input=conv_layer_12,\n",
|
||
" input_channels=512,\n",
|
||
" output_channels=512,\n",
|
||
" kernel_size=[3, 3],\n",
|
||
" strides=[1, 1, 1, 1],\n",
|
||
" scope='conv13') \n",
|
||
" \n",
|
||
" pool_layer_5 = tf.nn.max_pool(conv_layer_12,\n",
|
||
" ksize=[1, 2, 2, 1], \n",
|
||
" strides=[1, 2, 2, 1],\n",
|
||
" padding='SAME',\n",
|
||
" name='pool5') \n",
|
||
" # ===========\n",
|
||
" # CLASSIFIER\n",
|
||
" # ===========\n",
|
||
" \n",
|
||
" fc_layer_1 = fc_layer(input=pool_layer_5, \n",
|
||
" output_nodes=4096,\n",
|
||
" activation=tf.nn.relu,\n",
|
||
" scope='fc1')\n",
|
||
" \n",
|
||
" fc_layer_2 = fc_layer(input=fc_layer_1, \n",
|
||
" output_nodes=4096,\n",
|
||
" activation=tf.nn.relu,\n",
|
||
" scope='fc2')\n",
|
||
"\n",
|
||
" out_layer = fc_layer(input=fc_layer_2, \n",
|
||
" output_nodes=n_classes,\n",
|
||
" activation=None,\n",
|
||
" scope='output_layer')\n",
|
||
" \n",
|
||
" # Loss and optimizer\n",
|
||
" loss = tf.nn.softmax_cross_entropy_with_logits(logits=out_layer, labels=tf_y)\n",
|
||
" cost = tf.reduce_mean(loss, name='cost')\n",
|
||
" \n",
|
||
" optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
|
||
" train = optimizer.minimize(cost, name='train')\n",
|
||
"\n",
|
||
" # Prediction\n",
|
||
" correct_prediction = tf.equal(tf.argmax(tf_y, 1), tf.argmax(out_layer, 1), \n",
|
||
" name='correct_predictions')\n",
|
||
" accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n",
|
||
"\n",
|
||
" # Saver to save session for reuse\n",
|
||
" saver = tf.train.Saver()\n",
|
||
"\n",
|
||
" \n",
|
||
"##########################\n",
|
||
"### TRAINING & EVALUATION\n",
|
||
"##########################\n",
|
||
"\n",
|
||
"with tf.Session(graph=g) as sess:\n",
|
||
" sess.run(tf.global_variables_initializer())\n",
|
||
"\n",
|
||
" for epoch in range(training_epochs):\n",
|
||
" \n",
|
||
" avg_cost = 0.\n",
|
||
" mbatch_cnt = 0\n",
|
||
" for batch_x, batch_y in cifar.load_train_epoch(shuffle=True, batch_size=batch_size):\n",
|
||
" \n",
|
||
" mbatch_cnt += 1\n",
|
||
" _, c = sess.run(['train', 'cost:0'], feed_dict={'features:0': batch_x,\n",
|
||
" 'targets:0': batch_y})\n",
|
||
" avg_cost += c\n",
|
||
"\n",
|
||
" if not mbatch_cnt % print_interval:\n",
|
||
" print(\"Minibatch: %04d | Cost: %.3f\" % (mbatch_cnt, c))\n",
|
||
" \n",
|
||
"\n",
|
||
" # ===================\n",
|
||
" # Training Accuracy\n",
|
||
" # ===================\n",
|
||
" n_predictions, n_correct = 0, 0\n",
|
||
" for batch_x, batch_y in cifar.load_train_epoch(batch_size=batch_size):\n",
|
||
" \n",
|
||
" p = sess.run('correct_predictions:0', \n",
|
||
" feed_dict={'features:0': batch_x,\n",
|
||
" 'targets:0': batch_y})\n",
|
||
" n_correct += np.sum(p)\n",
|
||
" n_predictions += p.shape[0]\n",
|
||
" train_acc = n_correct / n_predictions\n",
|
||
" \n",
|
||
" \n",
|
||
" # ===================\n",
|
||
" # Validation Accuracy\n",
|
||
" # ===================\n",
|
||
" #valid_acc = sess.run('accuracy:0', feed_dict={'features:0': X_valid,\n",
|
||
" # 'targets:0': y_valid})\n",
|
||
" # ---------------------------------------\n",
|
||
" # workaround for GPUs with <= 4 Gb memory\n",
|
||
" n_predictions, n_correct = 0, 0\n",
|
||
" indices = np.arange(y_valid.shape[0])\n",
|
||
" chunksize = 500\n",
|
||
" for start_idx in range(0, indices.shape[0] - chunksize + 1, chunksize):\n",
|
||
" index_slice = indices[start_idx:start_idx + chunksize]\n",
|
||
" p = sess.run('correct_predictions:0', \n",
|
||
" feed_dict={'features:0': X_valid[index_slice],\n",
|
||
" 'targets:0': y_valid[index_slice]})\n",
|
||
" n_correct += np.sum(p)\n",
|
||
" n_predictions += p.shape[0]\n",
|
||
" valid_acc = n_correct / n_predictions\n",
|
||
" # ---------------------------------------\n",
|
||
" \n",
|
||
" print(\"Epoch: %03d | AvgCost: %.3f\" % (epoch + 1, avg_cost / mbatch_cnt), end=\"\")\n",
|
||
" print(\" | Train/Valid ACC: %.3f/%.3f\" % (train_acc, valid_acc))\n",
|
||
" \n",
|
||
" saver.save(sess, save_path='./convnet-vgg16.ckpt')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"INFO:tensorflow:Restoring parameters from ./convnet-vgg16.ckpt\n",
|
||
"Test ACC: 0.772\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"##########################\n",
|
||
"### RELOAD & TEST\n",
|
||
"##########################\n",
|
||
"\n",
|
||
"with tf.Session(graph=g) as sess:\n",
|
||
" saver.restore(sess, save_path='./convnet-vgg16.ckpt')\n",
|
||
" \n",
|
||
" # test_acc = sess.run('accuracy:0', feed_dict={'features:0': X_test,\n",
|
||
" # 'targets:0': y_test})\n",
|
||
" # ---------------------------------------\n",
|
||
" # workaround for GPUs with <= 4 Gb memory\n",
|
||
" n_predictions, n_correct = 0, 0\n",
|
||
" indices = np.arange(y_test.shape[0])\n",
|
||
" chunksize = 500\n",
|
||
" for start_idx in range(0, indices.shape[0] - chunksize + 1, chunksize):\n",
|
||
" index_slice = indices[start_idx:start_idx + chunksize]\n",
|
||
" p = sess.run('correct_predictions:0', \n",
|
||
" feed_dict={'features:0': X_test[index_slice],\n",
|
||
" 'targets:0': y_test[index_slice]})\n",
|
||
" n_correct += np.sum(p)\n",
|
||
" n_predictions += p.shape[0]\n",
|
||
" test_acc = n_correct / n_predictions\n",
|
||
" # ---------------------------------------\n",
|
||
"\n",
|
||
" print('Test ACC: %.3f' % test_acc)"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.7.1"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 2
|
||
}
|