chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+152
@@ -0,0 +1,152 @@
|
|||||||
|
# Other
|
||||||
|
.pytorch-lightning_ipynb_new/
|
||||||
|
|
||||||
|
# Log files
|
||||||
|
pytorch-lightning_ipynb/mlp/logs/
|
||||||
|
pytorch-lightning_ipynb/cnn/logs/
|
||||||
|
|
||||||
|
# Datasets
|
||||||
|
png-files
|
||||||
|
*-ubyte*
|
||||||
|
pytorch-lightning_ipynb/*/data
|
||||||
|
pytorch_ipynb/viz/cnns/cats-and-dogs/dogs-vs-cats
|
||||||
|
pytorch_ipynb/gan/dogs-vs-cats
|
||||||
|
pytorch_ipynb/viz/cnns/cats-and-dogs/dogs-vs-cats
|
||||||
|
pytorch_ipynb/rnn/yelp_review_polarity_csv/
|
||||||
|
pytorch_ipynb/rnn/ag_news_csv/
|
||||||
|
pytorch_ipynb/rnn/amazon_review_polarity_csv/
|
||||||
|
HistoricalColor-ECCV2012*
|
||||||
|
AFAD-Lite
|
||||||
|
tarball*
|
||||||
|
pytorch_ipynb/rnn/.data/
|
||||||
|
pytorch_ipynb/rnn/.vector_cache/
|
||||||
|
cifar-10-batches-py
|
||||||
|
celeba_gender_attr_test.txt
|
||||||
|
celeba_gender_attr_train.txt
|
||||||
|
iris.h5
|
||||||
|
test_32x32.mat
|
||||||
|
train_32x32.mat
|
||||||
|
code/model_zoo/pytorch_ipynb/svhn_cropped/
|
||||||
|
list_attr_celeba.txt
|
||||||
|
list_eval_partition.txt
|
||||||
|
img_align_celeba
|
||||||
|
quickdraw-*
|
||||||
|
*.csv
|
||||||
|
*.zip
|
||||||
|
*.npz
|
||||||
|
*.npy
|
||||||
|
*.tar.gz
|
||||||
|
*ubyte.gz
|
||||||
|
*archive.ics.uci.edu*
|
||||||
|
code/model_zoo/cifar-10
|
||||||
|
code/model_zoo/pytorch_ipynb/data
|
||||||
|
|
||||||
|
# Binary PyTorch models
|
||||||
|
*.pt
|
||||||
|
*.state_dict
|
||||||
|
|
||||||
|
# Temporary OS files
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# TensorFlow Checkpoint files
|
||||||
|
checkpoint
|
||||||
|
code/*/*.data-?????-of-?????
|
||||||
|
code/*/*.index
|
||||||
|
code/*/*.meta
|
||||||
|
code/model_zoo/tensorflow_ipynb/*.data-?????-of-?????
|
||||||
|
code/model_zoo/tensorflow_ipynb/*.index
|
||||||
|
code/model_zoo/tensorflow_ipynb/*.meta
|
||||||
|
code/model_zoo/tensorflow_ipynb/cifar-10/*
|
||||||
|
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
env/
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*,cover
|
||||||
|
.hypothesis/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
target/
|
||||||
|
|
||||||
|
# IPython Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
.python-version
|
||||||
|
|
||||||
|
# celery beat schedule file
|
||||||
|
celerybeat-schedule
|
||||||
|
|
||||||
|
# dotenv
|
||||||
|
.env
|
||||||
|
|
||||||
|
# virtualenv
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# Datasets
|
||||||
|
MNIST*
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# File : .pep8speaks.yml
|
||||||
|
|
||||||
|
scanner:
|
||||||
|
diff_only: True # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned.
|
||||||
|
linter: flake8
|
||||||
|
|
||||||
|
flake8:
|
||||||
|
max-line-length: 89 # Default is 79 in PEP 8
|
||||||
|
ignore: # Errors and warnings to ignore
|
||||||
|
- W504 # line break after binary operator
|
||||||
|
- E402 # module level import not at top of file
|
||||||
|
- E731 # do not assign a lambda expression, use a def
|
||||||
|
- C406 # Unnecessary list literal - rewrite as a dict literal.
|
||||||
|
- E741 # ambiguous variable name
|
||||||
|
|
||||||
|
no_blank_comment: False # If True, no comment is made on PR without any errors.
|
||||||
|
descending_issues_order: False # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file
|
||||||
|
|
||||||
|
message: # Customize the comment made by the bot
|
||||||
|
opened: # Messages when a new PR is submitted
|
||||||
|
header: "Hello @{name}! Thanks for opening this PR. "
|
||||||
|
# The keyword {name} is converted into the author's username
|
||||||
|
footer: ""
|
||||||
|
# The messages can be written as they would over GitHub
|
||||||
|
updated: # Messages when new commits are added to the PR
|
||||||
|
header: "Hello @{name}! Thanks for updating this PR. "
|
||||||
|
footer: "" # Why to comment the link to the style guide everytime? :)
|
||||||
|
no_errors: "There are currently no PEP 8 issues detected in this Pull Request. Cheers! :beers: "
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2019-2022 Sebastian Raschka
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
@@ -0,0 +1,427 @@
|
|||||||
|
|
||||||
|
# Deep Learning Models
|
||||||
|
|
||||||
|
A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Traditional Machine Learning
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Perceptron | 2D toy data | TBD | [](pytorch_ipynb/basic-ml/perceptron.ipynb) [](tensorflow1_ipynb/basic-ml/perceptron.ipynb) |
|
||||||
|
| Logistic Regression | 2D toy data | TBD | [](pytorch_ipynb/basic-ml/logistic-regression.ipynb) [](tensorflow1_ipynb/basic-ml/logistic-regression.ipynb)|
|
||||||
|
| Softmax Regression (Multinomial Logistic Regression) | MNIST | TBD | [](pytorch_ipynb/basic-ml/softmax-regression.ipynb) [](tensorflow1_ipynb/basic-ml/softmax-regression.ipynb) |
|
||||||
|
| Softmax Regression with MLxtend's plot_decision_regions on Iris | Iris | TBD | [](pytorch_ipynb/basic-ml/softmax-regression-mlxtend-1.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
## Multilayer Perceptrons
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Multilayer Perceptron | MNIST | TBD | [](pytorch-lightning_ipynb/mlp/mlp-basic.ipynb) [](pytorch_ipynb/mlp/mlp-basic.ipynb) [](tensorflow1_ipynb/mlp/mlp-basic.ipynb) |
|
||||||
|
| Multilayer Perceptron with Dropout | MNIST | TBD | [](pytorch-lightning_ipynb/mlp/mlp-dropout.ipynb) [](pytorch_ipynb/mlp/mlp-dropout.ipynb) [](tensorflow1_ipynb/mlp/mlp-dropout.ipynb) |
|
||||||
|
|Multilayer Perceptron with Batch Normalization | MNIST | TBD | [](pytorch-lightning_ipynb/mlp/mlp-batchnorm.ipynb) [](pytorch_ipynb/mlp/mlp-batchnorm.ipynb) [](tensorflow1_ipynb/mlp/mlp-batchtnorm.ipynb) |
|
||||||
|
|Multilayer Perceptron with Backpropagation from Scratch | MNIST | TBD | [](pytorch_ipynb/mlp/mlp-fromscratch__sigmoid-mse.ipynb) [](tensorflow1_ipynb/mlp/mlp-fromscratch__sigmoid-mse.ipynb)|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Convolutional Neural Networks
|
||||||
|
|
||||||
|
|
||||||
|
#### Basic
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Convolutional Neural Network | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-basic.ipynb) [](pytorch_ipynb/cnn/cnn-basic.ipynb) [](tensorflow1_ipynb/cnn/cnn-basic.ipynb) |
|
||||||
|
| CNN with He Initialization | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-he-init.ipynb) [](pytorch_ipynb/cnn/cnn-he-init.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Concepts
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Replacing Fully-Connected by Equivalent Convolutional Layers | TBD | TBD | [](pytorch_ipynb/cnn/fc-to-conv.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
|
||||||
|
#### AlexNet
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| AlexNet Trained on CIFAR-10 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-alexnet-cifar10.ipynb) [](pytorch_ipynb/cnn/cnn-alexnet-cifar10.ipynb) |
|
||||||
|
| AlexNet with Grouped Convolutions Trained on CIFAR-10 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-alexnet-grouped-cifar10.ipynb) [](pytorch_ipynb/cnn/cnn-alexnet-grouped-cifar10.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### DenseNet
|
||||||
|
|
||||||
|
|Title | Description | Daset | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| DenseNet-121 Digit Classifier Trained on MNIST | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-densenet121-mnist.ipynb) [](pytorch_ipynb/cnn/cnn-densenet121-mnist.ipynb) |
|
||||||
|
| DenseNet-121 Image Classifier Trained on CIFAR-10 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-densenet121-cifar10.ipynb) [](pytorch_ipynb/cnn/cnn-densenet121-cifar10.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Fully Convolutional
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| "All Convolutionl Net" -- A Fully Convolutional Neural Network | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-allconv.ipynb) [](pytorch_ipynb/cnn/cnn-allconv.ipynb) |
|
||||||
|
|
||||||
|
#### LeNet
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| LeNet-5 on MNIST | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-lenet5-mnist.ipynb) [](pytorch_ipynb/cnn/cnn-lenet5-mnist.ipynb) |
|
||||||
|
| LeNet-5 on CIFAR-10 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-lenet5-cifar10.ipynb) [](pytorch_ipynb/cnn/cnn-lenet5-cifar10.ipynb) |
|
||||||
|
| LeNet-5 on QuickDraw | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-lenet5-quickdraw.ipynb) [](pytorch_ipynb/cnn/cnn-lenet5-quickdraw.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### MobileNet
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| MobileNet-v2 on Cifar-10 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-mobilenet-v2-cifar10.ipynb) [](pytorch_ipynb/cnn/cnn-mobilenet-v2-cifar10.ipynb) |
|
||||||
|
| MobileNet-v3 small on Cifar-10 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-mobilenet-v3-small-cifar10.ipynb) [](pytorch_ipynb/cnn/cnn-mobilenet-v3-small-cifar10.ipynb) |
|
||||||
|
| MobileNet-v3 large on Cifar-10 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-mobilenet-v3-large-cifar10.ipynb) [](pytorch_ipynb/cnn/cnn-mobilenet-v3-large-cifar10.ipynb) |
|
||||||
|
| MobileNet-v3 large on MNIST via Embetter | TBD | TBD | [](pytorch_ipynb/cnn/cnn-embetter-mobilenet.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Network in Network
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Network in Network Trained on CIFAR-10 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-nin-cifar10.ipynb) [](pytorch_ipynb/cnn/nin-cifar10.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
#### VGG
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Convolutional Neural Network VGG-16 Trained on CIFAR-10 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-vgg16.ipynb) [](pytorch_ipynb/cnn/cnn-vgg16.ipynb) [](tensorflow1_ipynb/cnn/cnn-vgg16.ipynb) |
|
||||||
|
| VGG-16 Smile Classifier | [CelebA](https://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) | TBD | [](pytorch-lightning_ipynb/cnn/cnn-vgg16-celeba.ipynb) [](pytorch_ipynb/cnn/cnn-vgg16-celeba.ipynb) |
|
||||||
|
| VGG-16 Dogs vs Cats Classifier | TBD | TBD | [](pytorch_ipynb/cnn/cnn-vgg16-cats-dogs.ipynb) |
|
||||||
|
| Convolutional Neural Network VGG-19 | TBD | TBD | [](pytorch-lightning_ipynb/cnn/cnn-vgg19.ipynb) [](pytorch_ipynb/cnn/cnn-vgg19.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### ResNet
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| ResNet and Residual Blocks | [MNIST](http://yann.lecun.com/exdb/mnist/) | TBD | [](pytorch_ipynb/cnn/resnet-ex-1.ipynb) |
|
||||||
|
| ResNet-18 Digit Classifier| [MNIST](http://yann.lecun.com/exdb/mnist/) | TBD | [](pytorch_ipynb/cnn/cnn-resnet18-mnist.ipynb) |
|
||||||
|
| ResNet-18 Gender Classifier | [CelebA](https://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) | TBD | [](pytorch_ipynb/cnn/cnn-resnet18-celeba-dataparallel.ipynb) |
|
||||||
|
| ResNet-34 Digit Classifier | [MNIST](http://yann.lecun.com/exdb/mnist/) | TBD | [](pytorch_ipynb/cnn/cnn-resnet34-mnist.ipynb) |
|
||||||
|
| ResNet-34 Object Classifier | [QuickDraw](https://quickdraw.withgoogle.com) | TBD | [](pytorch_ipynb/cnn/cnn-resnet34-quickdraw.ipynb) |
|
||||||
|
| ResNet-34 Gender Classifier| [CelebA](https://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) | TBD | [](pytorch_ipynb/cnn/cnn-resnet34-celeba-dataparallel.ipynb) |
|
||||||
|
| ResNet-50 Digit Classifier| [MNIST](http://yann.lecun.com/exdb/mnist/) | TBD | [](pytorch_ipynb/cnn/cnn-resnet50-mnist.ipynb) |
|
||||||
|
| ResNet-50 Gender Classifier | [CelebA](https://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) | TBD | [](pytorch_ipynb/cnn/cnn-resnet50-celeba-dataparallel.ipynb) |
|
||||||
|
| ResNet-101 Gender Classifier| [CelebA](https://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) | TBD | [](pytorch_ipynb/cnn/cnn-resnet101-celeba.ipynb) |
|
||||||
|
| ResNet-101| [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html) | TBD | [](pytorch_ipynb/cnn/cnn-resnet101-cifar10.ipynb) |
|
||||||
|
| ResNet-152 Gender Classifier| [CelebA](https://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) | TBD | [](pytorch_ipynb/cnn/cnn-resnet152-celeba.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Transformers
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Multilabel DistilBERT | [Jigsaw Toxic Comment Challenge](https://www.kaggle.com/competitions/jigsaw-toxic-comment-classification-challenge) | DistilBERT classifier fine-tuning | [](pytorch_ipynb/transformer/distilbert-multilabel.ipynb) |
|
||||||
|
| DistilBERT as feature extractor | [IMDB movie review](https://ai.stanford.edu/~amaas/data/sentiment/) | DistilBERT classifier with sklearn random forest and logistic regression | [](pytorch_ipynb/transformer/1_distilbert-as-feature-extractor.ipynb) |
|
||||||
|
| DistilBERT as feature extractor using `embetter` | [IMDB movie review](https://ai.stanford.edu/~amaas/data/sentiment/) | DistilBERT classifier with sklearn random forest and logistic regression using the scikit-learn `embetter` library | [](pytorch_ipynb/transformer/distilbert-embetter-feature-extractor.ipynb) |
|
||||||
|
| Fine-tune DistilBERT I | [IMDB movie review](https://ai.stanford.edu/~amaas/data/sentiment/) | Fine-tune only the last 2 layers of DistilBERT classifier | [](pytorch-lightning_ipynb/transformer/distilbert-finetune-last-layers.ipynb) |
|
||||||
|
| Fine-tune DistilBERT II | [IMDB movie review](https://ai.stanford.edu/~amaas/data/sentiment/) | Fine-tune the whole DistilBERT classifier | [](pytorch_ipynb/transformer/distilbert-hf-finetuning.ipynb) [](pytorch-lightning_ipynb/transformer/distilbert-finetuning-ii.ipynb) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Ordinal Regression and Deep Learning
|
||||||
|
|
||||||
|
Please note that the following notebooks below provide reference implementations to use the respective methods. They are not performance benchmarks.
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Baseline multilayer perceptron | Cement | A baseline multilayer perceptron for classification trained with the standard cross entropy loss | [](pytorch_ipynb/ordinal/baseline_cement.ipynb) [](pytorch-lightning_ipynb/ordinal/baseline-light_cement.ipynb) |
|
||||||
|
| CORAL multilayer perceptron | Cement | Implementation of [Rank Consistent Ordinal Regression for Neural Networks with Application to Age Estimation](https://www.sciencedirect.com/science/article/pii/S016786552030413X) 2020 | [](pytorch_ipynb/ordinal/CORAL_cement.ipynb) [](pytorch-lightning_ipynb/ordinal/CORAL-light_cement.ipynb) |
|
||||||
|
| CORN multilayer perceptron | Cement | Implementation of [Deep Neural Networks for Rank-Consistent Ordinal Regression Based On Conditional Probabilities](https://arxiv.org/abs/2111.08851) 2022 | [](pytorch_ipynb/ordinal/CORN_cement.ipynb) [](pytorch-lightning_ipynb/ordinal/CORN-light_cement.ipynb) |
|
||||||
|
| Binary extension multilayer perceptron | Cement | Implementation of [Ordinal Regression with Multiple Output CNN for Age Estimation](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Niu_Ordinal_Regression_With_CVPR_2016_paper.pdf) 2016 | [](pytorch_ipynb/ordinal/niu2016_cement.ipynb) [](pytorch-lightning_ipynb/ordinal/niu2016-light_cement.ipynb) |
|
||||||
|
| Reformulated squared-error multilayer perceptron | Cement | Implementation of [A simple squared-error reformulation for ordinal classification](https://arxiv.org/abs/1612.00775) 2016 | [](pytorch_ipynb/ordinal/beckham2016_cement.ipynb) [](pytorch-lightning_ipynb/ordinal/beckham2016-light_cement.ipynb) |
|
||||||
|
| Class distance weighted cross-entropy loss | Cement | Implementation of [Class Distance Weighted Cross-Entropy Loss for Ulcerative Colitis Severity Estimation](https://arxiv.org/abs/2202.05167) 2022 | [](pytorch_ipynb/ordinal/polat2022_cement.ipynb) [](pytorch-lightning_ipynb/ordinal/polat2022-light_cement.ipynb) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Normalization Layers
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| BatchNorm before and after Activation for Network-in-Network CIFAR-10 Classifier | TBD | TBD | [](pytorch_ipynb/cnn/nin-cifar10_batchnorm.ipynb) |
|
||||||
|
| Filter Response Normalization for Network-in-Network CIFAR-10 Classifier | TBD | TBD | [](pytorch_ipynb/cnn/nin-cifar10_filter-response-norm.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Metric Learning
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Siamese Network with Multilayer Perceptrons | TBD | TBD | [](tensorflow1_ipynb/metric/siamese-1.ipynb) |
|
||||||
|
|
||||||
|
## Autoencoders
|
||||||
|
|
||||||
|
#### Fully-connected Autoencoders
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Autoencoder (MNIST) | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-basic.ipynb) [](tensorflow1_ipynb/autoencoder/ae-basic.ipynb) |
|
||||||
|
| Autoencoder (MNIST) + Scikit-Learn Random Forest Classifier | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-basic-with-rf.ipynb) [](tensorflow1_ipynb/autoencoder/ae-basic-with-rf.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Convolutional Autoencoders
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Convolutional Autoencoder with Deconvolutions / Transposed Convolutions | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-deconv.ipynb) [](tensorflow1_ipynb/autoencoder/ae-deconv.ipynb) |
|
||||||
|
| Convolutional Autoencoder with Deconvolutions and Continuous Jaccard Distance | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-deconv-jaccard.ipynb) |
|
||||||
|
| Convolutional Autoencoder with Deconvolutions (without pooling operations) | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-deconv-nopool.ipynb) |
|
||||||
|
| Convolutional Autoencoder with Nearest-neighbor Interpolation | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-conv-nneighbor.ipynb) [](tensorflow1_ipynb/autoencoder/ae-conv-nneighbor.ipynb) |
|
||||||
|
| Convolutional Autoencoder with Nearest-neighbor Interpolation -- Trained on CelebA | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-conv-nneighbor-celeba.ipynb) |
|
||||||
|
| Convolutional Autoencoder with Nearest-neighbor Interpolation -- Trained on Quickdraw | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-conv-nneighbor-quickdraw-1.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Variational Autoencoders
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Variational Autoencoder | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-var.ipynb) |
|
||||||
|
| Convolutional Variational Autoencoder | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-conv-var.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Conditional Variational Autoencoders
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Conditional Variational Autoencoder (with labels in reconstruction loss) | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-cvae.ipynb) |
|
||||||
|
| Conditional Variational Autoencoder (without labels in reconstruction loss) | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-cvae_no-out-concat.ipynb) |
|
||||||
|
| Convolutional Conditional Variational Autoencoder (with labels in reconstruction loss) | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-cnn-cvae.ipynb) |
|
||||||
|
| Convolutional Conditional Variational Autoencoder (without labels in reconstruction loss) | TBD | TBD | [](pytorch_ipynb/autoencoder/ae-cnn-cvae_no-out-concat.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Generative Adversarial Networks (GANs)
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Fully Connected GAN on MNIST | TBD | TBD | [](pytorch_ipynb/gan/gan.ipynb) [](tensorflow1_ipynb/gan/gan.ipynb) |
|
||||||
|
| Fully Connected Wasserstein GAN on MNIST | TBD | TBD | [](pytorch_ipynb/gan/wgan-1.ipynb) |
|
||||||
|
| Convolutional GAN on MNIST | TBD | TBD | [](pytorch_ipynb/gan/gan-conv.ipynb) [](tensorflow1_ipynb/gan/gan-conv.ipynb) |
|
||||||
|
| Convolutional GAN on MNIST with Label Smoothing | TBD | TBD | [](pytorch_ipynb/gan/gan-conv-smoothing.ipynb) [](tensorflow1_ipynb/gan/gan-conv-smoothing.ipynb) |
|
||||||
|
| Convolutional Wasserstein GAN on MNIST | TBD | TBD | [](pytorch_ipynb/gan/dc-wgan-1.ipynb) |
|
||||||
|
| Deep Convolutional GAN (DCGAN) on Cats and Dogs Images | TBD | TBD | [](pytorch_ipynb/gan/dcgan-cats-and-dogs.ipynb) |
|
||||||
|
| Deep Convolutional GAN (DCGAN) on CelebA Face Images | TBD | TBD | [](pytorch_ipynb/gan/dcgan-celeba.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
## Graph Neural Networks (GNNs)
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Most Basic Graph Neural Network with Gaussian Filter on MNIST | TBD | TBD | [](pytorch_ipynb/gnn/gnn-basic-1.ipynb) |
|
||||||
|
| Basic Graph Neural Network with Edge Prediction on MNIST | TBD | TBD | [](pytorch_ipynb/gnn/gnn-basic-edge-1.ipynb) |
|
||||||
|
| Basic Graph Neural Network with Spectral Graph Convolution on MNIST | TBD | TBD | [](pytorch_ipynb/gnn/gnn-basic-graph-spectral-1.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Recurrent Neural Networks (RNNs)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Many-to-one: Sentiment Analysis / Classification
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| A simple single-layer RNN (IMDB) | TBD | TBD | [](pytorch_ipynb/rnn/rnn_simple_imdb.ipynb) |
|
||||||
|
| A simple single-layer RNN with packed sequences to ignore padding characters (IMDB) | TBD | TBD | [](pytorch_ipynb/rnn/rnn_simple_packed_imdb.ipynb) |
|
||||||
|
| RNN with LSTM cells (IMDB) | TBD | TBD | [](pytorch_ipynb/rnn/rnn_lstm_packed_imdb.ipynb) |
|
||||||
|
| RNN with LSTM cells (IMDB) and pre-trained GloVe word vectors | TBD | TBD | [](pytorch_ipynb/rnn/rnn_lstm_packed_imdb-glove.ipynb) |
|
||||||
|
| RNN with LSTM cells and Own Dataset in CSV Format (IMDB) | TBD | TBD | [](pytorch_ipynb/rnn/rnn_lstm_packed_own_csv_imdb.ipynb) |
|
||||||
|
| RNN with GRU cells (IMDB) | TBD | TBD | [](pytorch_ipynb/rnn/rnn_gru_packed_imdb.ipynb) |
|
||||||
|
| Multilayer bi-directional RNN (IMDB) | TBD | TBD | [](pytorch_ipynb/rnn/rnn_lstm_bi_imdb.ipynb) |
|
||||||
|
| Bidirectional Multi-layer RNN with LSTM with Own Dataset in CSV Format (AG News) | TBD | TBD | [](pytorch_ipynb/rnn/rnn_bi_multilayer_lstm_own_csv_agnews.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Many-to-Many / Sequence-to-Sequence
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| A simple character RNN to generate new text (Charles Dickens) | TBD | TBD | [](pytorch_ipynb/rnn/char_rnn-charlesdickens.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Evaluation
|
||||||
|
|
||||||
|
### K-Fold Cross-Validation
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Baseline CNN | MNIST | A simple baseline with traditional train/validation/test splits | [](pytorch_ipynb/kfold/baseline-cnn-mnist.ipynb) [](pytorch-lightning_ipynb/kfold/baseline-light-cnn-mnist.ipynb) |
|
||||||
|
| K-fold with `pl_cross` | MNIST | A 5-fold cross-validation run using the `pl_cross` library | [](pytorch-lightning_ipynb/kfold/kfold-light-cnn-mnist.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Data Augmentation
|
||||||
|
|
||||||
|
| Title | Dataset | Description | Notebooks |
|
||||||
|
| -------------------------- | ------- | ----------- | ------------------------------------------------------------ |
|
||||||
|
| AutoAugment & TrivialAugment for Image Data | CIFAR-10 | Trains a ResNet-18 using AutoAugment and TrivialAugment | [](./pytorch-lightning_ipynb/data-augmentation/autoaugment) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Tips and Tricks
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Cyclical Learning Rate | TBD | TBD | [](pytorch_ipynb/tricks/cyclical-learning-rate.ipynb) |
|
||||||
|
| Annealing with Increasing the Batch Size (w. CIFAR-10 & AlexNet) | TBD | TBD | [](pytorch_ipynb/tricks/cnn-alexnet-cifar10-batchincrease.ipynb) |
|
||||||
|
| Gradient Clipping (w. MLP on MNIST) | TBD | TBD | [](pytorch_ipynb/tricks/gradclipping_mlp.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Transfer Learning
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Transfer Learning Example (VGG16 pre-trained on ImageNet for Cifar-10) | TBD | TBD | [](pytorch_ipynb/transfer/transferlearning-vgg16-cifar10-1.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
## Visualization and Interpretation
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Vanilla Loss Gradient (wrt Inputs) Visualization (Based on a VGG16 Convolutional Neural Network for Kaggle's Cats and Dogs Images) | TBD | TBD | [](pytorch_ipynb/viz/cnns/cats-and-dogs/cnn-viz-grad__vgg16-cats-dogs.ipynb) |
|
||||||
|
| Guided Backpropagation (Based on a VGG16 Convolutional Neural Network for Kaggle's Cats and Dogs Images) | TBD | TBD | [](pytorch_ipynb/viz/cnns/cats-and-dogs/cnn-viz-guided-backprop__vgg16-cats-dogs.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## PyTorch Workflows and Mechanics
|
||||||
|
|
||||||
|
|
||||||
|
#### PyTorch Lightning Examples
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| MLP in Lightning with TensorBoard -- continue training the last model | TBD | TBD | [](pytorch_ipynb/lightning/lightning-mlp.ipynb) |
|
||||||
|
| MLP in Lightning with TensorBoard -- checkpointing best model | TBD | TBD | [](pytorch_ipynb/lightning/lightning-mlp-best-model) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Custom Datasets
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Custom Data Loader Example for PNG Files | TBD | TBD | [](pytorch_ipynb/mechanics/custom-dataloader-png/custom-dataloader-example.ipynb) |
|
||||||
|
| Using PyTorch Dataset Loading Utilities for Custom Datasets -- CSV files converted to HDF5 | TBD | TBD | [](pytorch_ipynb/mechanics/custom-data-loader-csv.ipynb) |
|
||||||
|
| Using PyTorch Dataset Loading Utilities for Custom Datasets -- Face Images from CelebA | TBD | TBD | [](pytorch_ipynb/mechanics/custom-data-loader-celeba.ipynb) |
|
||||||
|
| Using PyTorch Dataset Loading Utilities for Custom Datasets -- Drawings from Quickdraw | TBD | TBD | [](pytorch_ipynb/mechanics/custom-data-loader-quickdraw.ipynb) |
|
||||||
|
| Using PyTorch Dataset Loading Utilities for Custom Datasets -- Drawings from the Street View House Number (SVHN) Dataset | TBD | TBD | [](pytorch_ipynb/mechanics/custom-data-loader-svhn.ipynb) |
|
||||||
|
| Using PyTorch Dataset Loading Utilities for Custom Datasets -- Asian Face Dataset (AFAD) | TBD | TBD | [](pytorch_ipynb/mechanics/custom-data-loader-afad.ipynb) |
|
||||||
|
| Using PyTorch Dataset Loading Utilities for Custom Datasets -- Dating Historical Color Images | TBD | TBD | [](pytorch_ipynb/mechanics/custom-data-loader_dating-historical-color-images.ipynb) |
|
||||||
|
| Using PyTorch Dataset Loading Utilities for Custom Datasets -- Fashion MNIST | TBD | TBD | [](pytorch_ipynb/mechanics/custom-data-loader-quickdraw.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Training and Preprocessing
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| PyTorch DataLoader State and Nested Iterations | Toy | Explains DataLoader behavior when in nested functions | [](pytorch_ipynb/mechanics/dataloader-nesting.ipynb)|
|
||||||
|
| Generating Validation Set Splits | TBD | TBD | [](pytorch_ipynb/mechanics/validation-splits.ipynb) |
|
||||||
|
| Dataloading with Pinned Memory | TBD | TBD | [](pytorch_ipynb/cnn/cnn-resnet34-cifar10-pinmem.ipynb) |
|
||||||
|
| Standardizing Images | TBD | TBD | [](pytorch_ipynb/cnn/cnn-standardized.ipynb) |
|
||||||
|
| Image Transformation Examples | TBD | TBD | [](pytorch_ipynb/mechanics/torchvision-transform-examples.ipynb) |
|
||||||
|
| Char-RNN with Own Text File | TBD | TBD | [](pytorch_ipynb/rnn/char_rnn-charlesdickens.ipynb) |
|
||||||
|
| Sentiment Classification RNN with Own CSV File | TBD | TBD | [](pytorch_ipynb/rnn/rnn_lstm_packed_own_csv_imdb.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Improving Memory Efficiency
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Gradient Checkpointing Demo (Network-in-Network trained on CIFAR-10) | TBD | TBD | [](pytorch_ipynb/mechanics/gradient-checkpointing-nin.ipynb) |
|
||||||
|
|
||||||
|
#### Parallel Computing
|
||||||
|
|
||||||
|
|Title | Description | Notebooks |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Using Multiple GPUs with DataParallel -- VGG-16 Gender Classifier on CelebA | TBD | [](pytorch_ipynb/cnn/cnn-vgg16-celeba-data-parallel.ipynb) |
|
||||||
|
| Distribute a Model Across Multiple GPUs with Pipeline Parallelism (VGG-16 Example) | TBD | [](pytorch_ipynb/mechanics/model-pipeline-vgg16.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
#### Other
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| PyTorch with and without Deterministic Behavior -- Runtime Benchmark | TBD | TBD | [](pytorch_ipynb/mechanics/deterministic_benchmark.ipynb) |
|
||||||
|
| Sequential API and hooks | TBD | TBD | [](pytorch_ipynb/mechanics/mlp-sequential.ipynb) |
|
||||||
|
| Weight Sharing Within a Layer | TBD | TBD | [](pytorch_ipynb/mechanics/cnn-weight-sharing.ipynb) |
|
||||||
|
| Plotting Live Training Performance in Jupyter Notebooks with just Matplotlib | TBD | TBD | [](pytorch_ipynb/mechanics/plot-jupyter-matplotlib.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Autograd
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Getting Gradients of an Intermediate Variable in PyTorch | TBD | TBD | [](pytorch_ipynb/mechanics/manual-gradients.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
## TensorFlow Workflows and Mechanics
|
||||||
|
|
||||||
|
#### Custom Datasets
|
||||||
|
|
||||||
|
|Title | Description | Notebooks |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Chunking an Image Dataset for Minibatch Training using NumPy NPZ Archives | TBD | [](tensorflow1_ipynb/mechanics/image-data-chunking-npz.ipynb) |
|
||||||
|
| Storing an Image Dataset for Minibatch Training using HDF5 | TBD | [](tensorflow1_ipynb/mechanics/image-data-chunking-hdf5.ipynb) |
|
||||||
|
| Using Input Pipelines to Read Data from TFRecords Files | TBD | [](tensorflow1_ipynb/mechanics/tfrecords.ipynb) |
|
||||||
|
| Using Queue Runners to Feed Images Directly from Disk | TBD | [](tensorflow1_ipynb/mechanics/file-queues.ipynb) |
|
||||||
|
| Using TensorFlow's Dataset API | TBD | [](tensorflow1_ipynb/mechanics/dataset-api.ipynb) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Training and Preprocessing
|
||||||
|
|
||||||
|
|Title | Dataset | Description | Notebooks |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Saving and Loading Trained Models -- from TensorFlow Checkpoint Files and NumPy NPZ Archives | TBD | TBD | [](tensorflow1_ipynb/mechanics/saving-and-reloading-models.ipynb) |
|
||||||
|
|
||||||
|
## Related Libraries
|
||||||
|
|
||||||
|
|Title | Description | Notebooks |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| TorchMetrics | How do we use it, and what's the difference between .update() and .forward()? | [](pytorch_ipynb/related-libraries/torchmetrics-update-forward.ipynb) |
|
||||||
|
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
# WeHub 来源说明
|
||||||
|
|
||||||
|
- 原始项目：`rasbt/deeplearning-models`
|
||||||
|
- 原始仓库：https://github.com/rasbt/deeplearning-models
|
||||||
|
- 导入方式：上游默认分支的最新快照
|
||||||
|
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||||
|
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,195 @@
|
|||||||
|
import lightning as L
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
import pandas as pd
|
||||||
|
import torch
|
||||||
|
import torch.nn.functional as F
|
||||||
|
import torchmetrics
|
||||||
|
from torch.utils.data import DataLoader
|
||||||
|
from torch.utils.data.dataset import random_split
|
||||||
|
from torchvision import datasets, transforms
|
||||||
|
|
||||||
|
|
||||||
|
class LightningModel(L.LightningModule):
|
||||||
|
def __init__(self, model, learning_rate):
|
||||||
|
super().__init__()
|
||||||
|
|
||||||
|
self.learning_rate = learning_rate
|
||||||
|
self.model = model
|
||||||
|
|
||||||
|
self.save_hyperparameters(ignore=["model"])
|
||||||
|
|
||||||
|
self.train_acc = torchmetrics.Accuracy(task="multiclass", num_classes=10)
|
||||||
|
self.val_acc = torchmetrics.Accuracy(task="multiclass", num_classes=10)
|
||||||
|
self.test_acc = torchmetrics.Accuracy(task="multiclass", num_classes=10)
|
||||||
|
|
||||||
|
def forward(self, x):
|
||||||
|
return self.model(x)
|
||||||
|
|
||||||
|
def _shared_step(self, batch):
|
||||||
|
features, true_labels = batch
|
||||||
|
logits = self(features)
|
||||||
|
|
||||||
|
loss = F.cross_entropy(logits, true_labels)
|
||||||
|
predicted_labels = torch.argmax(logits, dim=1)
|
||||||
|
return loss, true_labels, predicted_labels
|
||||||
|
|
||||||
|
def training_step(self, batch, batch_idx):
|
||||||
|
loss, true_labels, predicted_labels = self._shared_step(batch)
|
||||||
|
|
||||||
|
self.log("train_loss", loss)
|
||||||
|
self.train_acc(predicted_labels, true_labels)
|
||||||
|
self.log(
|
||||||
|
"train_acc", self.train_acc, prog_bar=True, on_epoch=True, on_step=False
|
||||||
|
)
|
||||||
|
return loss
|
||||||
|
|
||||||
|
def validation_step(self, batch, batch_idx):
|
||||||
|
loss, true_labels, predicted_labels = self._shared_step(batch)
|
||||||
|
|
||||||
|
self.log("val_loss", loss, prog_bar=True)
|
||||||
|
self.val_acc(predicted_labels, true_labels)
|
||||||
|
self.log("val_acc", self.val_acc, prog_bar=True)
|
||||||
|
|
||||||
|
def test_step(self, batch, batch_idx):
|
||||||
|
loss, true_labels, predicted_labels = self._shared_step(batch)
|
||||||
|
self.test_acc(predicted_labels, true_labels)
|
||||||
|
self.log("test_acc", self.test_acc)
|
||||||
|
|
||||||
|
def configure_optimizers(self):
|
||||||
|
optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate)
|
||||||
|
return optimizer
|
||||||
|
|
||||||
|
|
||||||
|
class Cifar10DataModule(L.LightningDataModule):
|
||||||
|
def __init__(
|
||||||
|
self, data_path="./", batch_size=64, num_workers=0, height_width=(32, 32),
|
||||||
|
train_transform=None, test_transform=None
|
||||||
|
):
|
||||||
|
super().__init__()
|
||||||
|
self.batch_size = batch_size
|
||||||
|
self.data_path = data_path
|
||||||
|
self.num_workers = num_workers
|
||||||
|
self.height_width = height_width
|
||||||
|
self.train_transform = train_transform
|
||||||
|
self.test_transform = test_transform
|
||||||
|
|
||||||
|
def prepare_data(self):
|
||||||
|
datasets.CIFAR10(root=self.data_path, download=True)
|
||||||
|
|
||||||
|
if self.train_transform is None:
|
||||||
|
self.train_transform = transforms.Compose(
|
||||||
|
[
|
||||||
|
transforms.Resize(self.height_width),
|
||||||
|
transforms.ToTensor(),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
if self.test_transform is None:
|
||||||
|
self.test_transform = transforms.Compose(
|
||||||
|
[
|
||||||
|
transforms.Resize(self.height_width),
|
||||||
|
transforms.ToTensor(),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
def setup(self, stage=None):
|
||||||
|
train = datasets.CIFAR10(
|
||||||
|
root=self.data_path,
|
||||||
|
train=True,
|
||||||
|
transform=self.train_transform,
|
||||||
|
download=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.test = datasets.CIFAR10(
|
||||||
|
root=self.data_path,
|
||||||
|
train=False,
|
||||||
|
transform=self.test_transform,
|
||||||
|
download=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.train, self.valid = random_split(train, lengths=[45000, 5000])
|
||||||
|
|
||||||
|
def train_dataloader(self):
|
||||||
|
train_loader = DataLoader(
|
||||||
|
dataset=self.train,
|
||||||
|
batch_size=self.batch_size,
|
||||||
|
drop_last=True,
|
||||||
|
shuffle=True,
|
||||||
|
num_workers=self.num_workers,
|
||||||
|
)
|
||||||
|
return train_loader
|
||||||
|
|
||||||
|
def val_dataloader(self):
|
||||||
|
valid_loader = DataLoader(
|
||||||
|
dataset=self.valid,
|
||||||
|
batch_size=self.batch_size,
|
||||||
|
drop_last=False,
|
||||||
|
shuffle=False,
|
||||||
|
num_workers=self.num_workers,
|
||||||
|
)
|
||||||
|
return valid_loader
|
||||||
|
|
||||||
|
def test_dataloader(self):
|
||||||
|
test_loader = DataLoader(
|
||||||
|
dataset=self.test,
|
||||||
|
batch_size=self.batch_size,
|
||||||
|
drop_last=False,
|
||||||
|
shuffle=False,
|
||||||
|
num_workers=self.num_workers,
|
||||||
|
)
|
||||||
|
return test_loader
|
||||||
|
|
||||||
|
|
||||||
|
def plot_val_acc(
|
||||||
|
log_dir, acc_ylim=(0.5, 1.0), save_loss=None, save_acc=None):
|
||||||
|
|
||||||
|
metrics = pd.read_csv(f"{log_dir}/metrics.csv")
|
||||||
|
|
||||||
|
aggreg_metrics = []
|
||||||
|
agg_col = "epoch"
|
||||||
|
|
||||||
|
for i, dfg in metrics.groupby(agg_col):
|
||||||
|
agg = dict(dfg.mean())
|
||||||
|
agg[agg_col] = i
|
||||||
|
aggreg_metrics.append(agg)
|
||||||
|
|
||||||
|
df_metrics = pd.DataFrame(aggreg_metrics)
|
||||||
|
df_metrics[["val_acc"]].plot(
|
||||||
|
grid=True, legend=True, xlabel="Epoch", ylabel="ACC"
|
||||||
|
)
|
||||||
|
|
||||||
|
plt.ylim(acc_ylim)
|
||||||
|
if save_acc is not None:
|
||||||
|
plt.savefig(save_acc)
|
||||||
|
|
||||||
|
|
||||||
|
def plot_loss_and_acc(
|
||||||
|
log_dir, loss_ylim=(0.0, 0.9), acc_ylim=(0.3, 1.0), save_loss=None, save_acc=None
|
||||||
|
):
|
||||||
|
|
||||||
|
metrics = pd.read_csv(f"{log_dir}/metrics.csv")
|
||||||
|
|
||||||
|
aggreg_metrics = []
|
||||||
|
agg_col = "epoch"
|
||||||
|
for i, dfg in metrics.groupby(agg_col):
|
||||||
|
agg = dict(dfg.mean())
|
||||||
|
agg[agg_col] = i
|
||||||
|
aggreg_metrics.append(agg)
|
||||||
|
|
||||||
|
df_metrics = pd.DataFrame(aggreg_metrics)
|
||||||
|
df_metrics[["train_loss"]].plot(
|
||||||
|
grid=True, legend=True, xlabel="Epoch", ylabel="Loss"
|
||||||
|
)
|
||||||
|
|
||||||
|
plt.ylim(loss_ylim)
|
||||||
|
if save_loss is not None:
|
||||||
|
plt.savefig(save_loss)
|
||||||
|
|
||||||
|
df_metrics[["train_acc", "val_acc"]].plot(
|
||||||
|
grid=True, legend=True, xlabel="Epoch", ylabel="ACC"
|
||||||
|
)
|
||||||
|
|
||||||
|
plt.ylim(acc_ylim)
|
||||||
|
if save_acc is not None:
|
||||||
|
plt.savefig(save_acc)
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,541 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "1e2086ae",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The three extensions below are optional, for more information, see\n",
|
||||||
|
"- `watermark`: https://github.com/rasbt/watermark\n",
|
||||||
|
"- `pycodestyle_magic`: https://github.com/mattijn/pycodestyle_magic\n",
|
||||||
|
"- `nb_black`: https://github.com/dnanhkhoa/nb_black"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "149f964e",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -p torch,pytorch_lightning,torchmetrics,matplotlib"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "625d1d05",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load_ext pycodestyle_magic\n",
|
||||||
|
"%flake8_on --ignore W291,W293,E703,E402,E999 --max_line_length=100"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "f52f4600",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load_ext nb_black"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "9dfafe15",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"<a href=\"https://pytorch.org\"><img src=\"https://raw.githubusercontent.com/pytorch/pytorch/master/docs/source/_static/img/pytorch-logo-dark.svg\" width=\"90\"/></a> <a href=\"https://www.pytorchlightning.ai\"><img src=\"https://raw.githubusercontent.com/PyTorchLightning/pytorch-lightning/master/docs/source/_static/images/logo.svg\" width=\"150\"/></a>\n",
|
||||||
|
"\n",
|
||||||
|
"# TITLE"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "501948bd",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- DESCRIPTION\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"### References\n",
|
||||||
|
"\n",
|
||||||
|
"- ???"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "b7e16245",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## General settings and hyperparameters"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "b3915fdd",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Here, we specify some general hyperparameter values and general settings."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "7f540e31",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"BATCH_SIZE = 256\n",
|
||||||
|
"NUM_EPOCHS = 10\n",
|
||||||
|
"LEARNING_RATE = 0.005\n",
|
||||||
|
"NUM_WORKERS = 4"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "b472b8d9",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Note that using multiple workers can sometimes cause issues with too many open files in PyTorch for small datasets. If we have problems with the data loader later, try setting `NUM_WORKERS = 0` and reload the notebook."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "a85efca2",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Implementing a Neural Network using PyTorch Lightning's `LightningModule`"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "041bfee2",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- In this section, we set up the main model architecture using the `LightningModule` from PyTorch Lightning.\n",
|
||||||
|
"- In essence, `LightningModule` is a wrapper around a PyTorch module.\n",
|
||||||
|
"- We start with defining our neural network model in pure PyTorch, and then we use it in the `LightningModule` to get all the extra benefits that PyTorch Lightning provides."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "1cda97b5",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# UNIQUE MODEL CODE"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "2d8c0e05",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_lightningmodule/lightningmodule_classifier_basic.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "2961a362",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Setting up the dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "d86805b0",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- In this section, we are going to set up our dataset."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "a6fa61b4",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Inspecting the dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "087f0762",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_dataset/dataset_???_check.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "b8305a40",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Performance baseline"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "d8c4d897",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Especially for imbalanced datasets, it's pretty helpful to compute a performance baseline.\n",
|
||||||
|
"- In classification contexts, a useful baseline is to compute the accuracy for a scenario where the model always predicts the majority class -- we want our model to be better than that!"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "993ad0d5",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_dataset/performance_baseline.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "dab874ca",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## A quick visual check"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "3a6f0aa6",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_dataset/plot_visual-check_basic.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "4f40449e",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Setting up a `DataModule`"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "734560ae",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- There are three main ways we can prepare the dataset for Lightning. We can\n",
|
||||||
|
" 1. make the dataset part of the model;\n",
|
||||||
|
" 2. set up the data loaders as usual and feed them to the fit method of a Lightning Trainer -- the Trainer is introduced in the following subsection;\n",
|
||||||
|
" 3. create a LightningDataModule.\n",
|
||||||
|
"- Here, we will use approach 3, which is the most organized approach. The `LightningDataModule` consists of several self-explanatory methods, as we can see below:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "f5c8cf5b",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_lightningmodule/datamodule_???_basic.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "25132aa4",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Note that the `prepare_data` method is usually used for steps that only need to be executed once, for example, downloading the dataset; the `setup` method defines the dataset loading -- if we run our code in a distributed setting, this will be called on each node / GPU. \n",
|
||||||
|
"- Next, let's initialize the `DataModule`; we use a random seed for reproducibility (so that the data set is shuffled the same way when we re-execute this code):"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "17132fed",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"torch.manual_seed(1) \n",
|
||||||
|
"data_module = DataModule(data_path='./data')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "3ad8293d",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Training the model using the PyTorch Lightning Trainer class"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "25c2b2fb",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Next, we initialize our model.\n",
|
||||||
|
"- Also, we define a call back to obtain the model with the best validation set performance after training.\n",
|
||||||
|
"- PyTorch Lightning offers [many advanced logging services](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) like Weights & Biases. However, here, we will keep things simple and use the `CSVLogger`:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "c1acb2f1",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"pytorch_model = PyTorchModel(\n",
|
||||||
|
" ???\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "0ca2ccdd",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_lightningmodule/logger_csv_acc_basic.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "64857a91",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Now it's time to train our model:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "61f9f037",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_lightningmodule/trainer_nb_basic.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "cb7a9818",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Evaluating the model"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "5556da91",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- After training, let's plot our training ACC and validation ACC using pandas, which, in turn, uses matplotlib for plotting (PS: you may want to check out [more advanced logger](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) later on, which take care of it for us):"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "b424d39d",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_lightningmodule/logger_csv_plot_basic.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "ee93a525",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- The `trainer` automatically saves the model with the best validation accuracy automatically for us, we which we can load from the checkpoint via the `ckpt_path='best'` argument; below we use the `trainer` instance to evaluate the best model on the test set:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "d02f1378",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"trainer.test(model=lightning_model, datamodule=data_module, ckpt_path='best')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "b6cd101f",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Predicting labels of new data"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "ffed918e",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- We can use the `trainer.predict` method either on a new `DataLoader` (`trainer.predict(dataloaders=...)`) or `DataModule` (`trainer.predict(datamodule=...)`) to apply the model to new data.\n",
|
||||||
|
"- Alternatively, we can also manually load the best model from a checkpoint as shown below:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "9e018ee1",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"path = trainer.checkpoint_callback.best_model_path\n",
|
||||||
|
"print(path)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "d2548be4",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"lightning_model = LightningModel.load_from_checkpoint(path, model=pytorch_model)\n",
|
||||||
|
"lightning_model.eval();"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "82eea81a",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- For simplicity, we reused our existing `pytorch_model` above. However, we could also reinitialize the `pytorch_model`, and the `.load_from_checkpoint` method would load the corresponding model weights for us from the checkpoint file.\n",
|
||||||
|
"- Now, below is an example applying the model manually. Here, pretend that the `test_dataloader` is a new data loader."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "e5781814",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_lightningmodule/datamodule_testloader.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "ad3600a0",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- As an internal check, if the model was loaded correctly, the test accuracy below should be identical to the test accuracy we saw earlier in the previous section."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "c31bbca9",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"test_acc = acc.compute()\n",
|
||||||
|
"print(f'Test accuracy: {test_acc:.4f} ({test_acc*100:.2f}%)')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "9e2265c7",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Inspecting Failure Cases"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "4913044c",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- In practice, it is often informative to look at failure cases like wrong predictions for particular training instances as it can give us some insights into the model behavior and dataset.\n",
|
||||||
|
"- Inspecting failure cases can sometimes reveal interesting patterns and even highlight dataset and labeling issues."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "d1b5c4a6",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# In the case of ???, the class label mapping\n",
|
||||||
|
"# ???\n",
|
||||||
|
"class_dict = {???}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "18ab0294",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_lightningmodule/plot_failurecases_basic.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "481b4f08",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- In addition to inspecting failure cases visually, it is also informative to look at which classes the model confuses the most via a confusion matrix:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "a2cb8af8",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load ../code_lightningmodule/plot_confusion-matrix_basic.py"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "8d48d1ef",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%watermark --iversions"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.8.12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+1150
File diff suppressed because it is too large
Load Diff
+1152
File diff suppressed because it is too large
Load Diff
+1161
File diff suppressed because it is too large
Load Diff
BIN
Binary file not shown.
|
After Width: | Height: | Size: 121 KiB |
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 121 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 121 KiB |
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,373 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.6.8\n",
|
||||||
|
"IPython 7.2.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.0.0\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Runs on CPU or GPU (if available)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- Softmax Regression"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Implementation of softmax regression (multinomial logistic regression)."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"import torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Settings and Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Image batch dimensions: torch.Size([256, 1, 28, 28])\n",
|
||||||
|
"Image label dimensions: torch.Size([256])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Device\n",
|
||||||
|
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"random_seed = 123\n",
|
||||||
|
"learning_rate = 0.1\n",
|
||||||
|
"num_epochs = 10\n",
|
||||||
|
"batch_size = 256\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"num_features = 784\n",
|
||||||
|
"num_classes = 10\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"train_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(), \n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"class SoftmaxRegression(torch.nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, num_features, num_classes):\n",
|
||||||
|
" super(SoftmaxRegression, self).__init__()\n",
|
||||||
|
" self.linear = torch.nn.Linear(num_features, num_classes)\n",
|
||||||
|
" \n",
|
||||||
|
" self.linear.weight.detach().zero_()\n",
|
||||||
|
" self.linear.bias.detach().zero_()\n",
|
||||||
|
" \n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" logits = self.linear(x)\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
"model = SoftmaxRegression(num_features=num_features,\n",
|
||||||
|
" num_classes=num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
"model.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
"##########################\n",
|
||||||
|
"### COST AND OPTIMIZER\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 000/234 | Cost: 2.3026\n",
|
||||||
|
"Epoch: 001/010 | Batch 050/234 | Cost: 0.7941\n",
|
||||||
|
"Epoch: 001/010 | Batch 100/234 | Cost: 0.5651\n",
|
||||||
|
"Epoch: 001/010 | Batch 150/234 | Cost: 0.4603\n",
|
||||||
|
"Epoch: 001/010 | Batch 200/234 | Cost: 0.4822\n",
|
||||||
|
"Epoch: 001/010 training accuracy: 88.04%\n",
|
||||||
|
"Epoch: 002/010 | Batch 000/234 | Cost: 0.4105\n",
|
||||||
|
"Epoch: 002/010 | Batch 050/234 | Cost: 0.4415\n",
|
||||||
|
"Epoch: 002/010 | Batch 100/234 | Cost: 0.4367\n",
|
||||||
|
"Epoch: 002/010 | Batch 150/234 | Cost: 0.4289\n",
|
||||||
|
"Epoch: 002/010 | Batch 200/234 | Cost: 0.3926\n",
|
||||||
|
"Epoch: 002/010 training accuracy: 89.37%\n",
|
||||||
|
"Epoch: 003/010 | Batch 000/234 | Cost: 0.4112\n",
|
||||||
|
"Epoch: 003/010 | Batch 050/234 | Cost: 0.3579\n",
|
||||||
|
"Epoch: 003/010 | Batch 100/234 | Cost: 0.3013\n",
|
||||||
|
"Epoch: 003/010 | Batch 150/234 | Cost: 0.3258\n",
|
||||||
|
"Epoch: 003/010 | Batch 200/234 | Cost: 0.4254\n",
|
||||||
|
"Epoch: 003/010 training accuracy: 89.98%\n",
|
||||||
|
"Epoch: 004/010 | Batch 000/234 | Cost: 0.3988\n",
|
||||||
|
"Epoch: 004/010 | Batch 050/234 | Cost: 0.3690\n",
|
||||||
|
"Epoch: 004/010 | Batch 100/234 | Cost: 0.3459\n",
|
||||||
|
"Epoch: 004/010 | Batch 150/234 | Cost: 0.4030\n",
|
||||||
|
"Epoch: 004/010 | Batch 200/234 | Cost: 0.3240\n",
|
||||||
|
"Epoch: 004/010 training accuracy: 90.35%\n",
|
||||||
|
"Epoch: 005/010 | Batch 000/234 | Cost: 0.3265\n",
|
||||||
|
"Epoch: 005/010 | Batch 050/234 | Cost: 0.3673\n",
|
||||||
|
"Epoch: 005/010 | Batch 100/234 | Cost: 0.3085\n",
|
||||||
|
"Epoch: 005/010 | Batch 150/234 | Cost: 0.3183\n",
|
||||||
|
"Epoch: 005/010 | Batch 200/234 | Cost: 0.3316\n",
|
||||||
|
"Epoch: 005/010 training accuracy: 90.64%\n",
|
||||||
|
"Epoch: 006/010 | Batch 000/234 | Cost: 0.4518\n",
|
||||||
|
"Epoch: 006/010 | Batch 050/234 | Cost: 0.3863\n",
|
||||||
|
"Epoch: 006/010 | Batch 100/234 | Cost: 0.3620\n",
|
||||||
|
"Epoch: 006/010 | Batch 150/234 | Cost: 0.3733\n",
|
||||||
|
"Epoch: 006/010 | Batch 200/234 | Cost: 0.3289\n",
|
||||||
|
"Epoch: 006/010 training accuracy: 90.86%\n",
|
||||||
|
"Epoch: 007/010 | Batch 000/234 | Cost: 0.3450\n",
|
||||||
|
"Epoch: 007/010 | Batch 050/234 | Cost: 0.2289\n",
|
||||||
|
"Epoch: 007/010 | Batch 100/234 | Cost: 0.3073\n",
|
||||||
|
"Epoch: 007/010 | Batch 150/234 | Cost: 0.2750\n",
|
||||||
|
"Epoch: 007/010 | Batch 200/234 | Cost: 0.3456\n",
|
||||||
|
"Epoch: 007/010 training accuracy: 91.00%\n",
|
||||||
|
"Epoch: 008/010 | Batch 000/234 | Cost: 0.4900\n",
|
||||||
|
"Epoch: 008/010 | Batch 050/234 | Cost: 0.3479\n",
|
||||||
|
"Epoch: 008/010 | Batch 100/234 | Cost: 0.2343\n",
|
||||||
|
"Epoch: 008/010 | Batch 150/234 | Cost: 0.3059\n",
|
||||||
|
"Epoch: 008/010 | Batch 200/234 | Cost: 0.3684\n",
|
||||||
|
"Epoch: 008/010 training accuracy: 91.22%\n",
|
||||||
|
"Epoch: 009/010 | Batch 000/234 | Cost: 0.3762\n",
|
||||||
|
"Epoch: 009/010 | Batch 050/234 | Cost: 0.2976\n",
|
||||||
|
"Epoch: 009/010 | Batch 100/234 | Cost: 0.2690\n",
|
||||||
|
"Epoch: 009/010 | Batch 150/234 | Cost: 0.2610\n",
|
||||||
|
"Epoch: 009/010 | Batch 200/234 | Cost: 0.3140\n",
|
||||||
|
"Epoch: 009/010 training accuracy: 91.34%\n",
|
||||||
|
"Epoch: 010/010 | Batch 000/234 | Cost: 0.2790\n",
|
||||||
|
"Epoch: 010/010 | Batch 050/234 | Cost: 0.3070\n",
|
||||||
|
"Epoch: 010/010 | Batch 100/234 | Cost: 0.3300\n",
|
||||||
|
"Epoch: 010/010 | Batch 150/234 | Cost: 0.2520\n",
|
||||||
|
"Epoch: 010/010 | Batch 200/234 | Cost: 0.3301\n",
|
||||||
|
"Epoch: 010/010 training accuracy: 91.40%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# Manual seed for deterministic data loader\n",
|
||||||
|
"torch.manual_seed(random_seed)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def compute_accuracy(model, data_loader):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" \n",
|
||||||
|
" for features, targets in data_loader:\n",
|
||||||
|
" features = features.view(-1, 28*28).to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" \n",
|
||||||
|
" return correct_pred.float() / num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"for epoch in range(num_epochs):\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.view(-1, 28*28).to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
" \n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" \n",
|
||||||
|
" # note that the PyTorch implementation of\n",
|
||||||
|
" # CrossEntropyLoss works with logits, not\n",
|
||||||
|
" # probabilities\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, num_epochs, batch_idx, \n",
|
||||||
|
" len(train_dataset)//batch_size, cost))\n",
|
||||||
|
" \n",
|
||||||
|
" with torch.set_grad_enabled(False):\n",
|
||||||
|
" print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
|
||||||
|
" epoch+1, num_epochs, \n",
|
||||||
|
" compute_accuracy(model, train_loader)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 91.77%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"torch 1.0.0\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.1"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": false,
|
||||||
|
"toc_position": {},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,517 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.6.8\n",
|
||||||
|
"IPython 7.2.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.0.1.post2\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Runs on CPU or GPU (if available)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- All-Convolutional Neural Network"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Simple convolutional neural network that uses stride=2 every 2nd convolutional layer, instead of max pooling, to reduce the feature maps. Loosely based on\n",
|
||||||
|
"\n",
|
||||||
|
"- Springenberg, Jost Tobias, Alexey Dosovitskiy, Thomas Brox, and Martin Riedmiller. \"Striving for simplicity: The all convolutional net.\" arXiv preprint arXiv:1412.6806 (2014)."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import time\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Settings and Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Image batch dimensions: torch.Size([256, 1, 28, 28])\n",
|
||||||
|
"Image label dimensions: torch.Size([256])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Device\n",
|
||||||
|
"device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"random_seed = 1\n",
|
||||||
|
"learning_rate = 0.001\n",
|
||||||
|
"num_epochs = 15\n",
|
||||||
|
"batch_size = 256\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"num_classes = 10\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Model"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class ConvNet(torch.nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, num_classes):\n",
|
||||||
|
" super(ConvNet, self).__init__()\n",
|
||||||
|
" \n",
|
||||||
|
" self.num_classes = num_classes\n",
|
||||||
|
" # calculate same padding:\n",
|
||||||
|
" # (w - k + 2*p)/s + 1 = o\n",
|
||||||
|
" # => p = (s(o-1) - w + k)/2\n",
|
||||||
|
" \n",
|
||||||
|
" # 28x28x1 => 28x28x4\n",
|
||||||
|
" self.conv_1 = torch.nn.Conv2d(in_channels=1,\n",
|
||||||
|
" out_channels=4,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(28-1) - 28 + 3) / 2 = 1\n",
|
||||||
|
" # 28x28x4 => 14x14x4\n",
|
||||||
|
" self.conv_2 = torch.nn.Conv2d(in_channels=4,\n",
|
||||||
|
" out_channels=4,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(2, 2),\n",
|
||||||
|
" padding=1) \n",
|
||||||
|
" # 14x14x4 => 14x14x8\n",
|
||||||
|
" self.conv_3 = torch.nn.Conv2d(in_channels=4,\n",
|
||||||
|
" out_channels=8,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(14-1) - 14 + 3) / 2 = 1 \n",
|
||||||
|
" # 14x14x8 => 7x7x8 \n",
|
||||||
|
" self.conv_4 = torch.nn.Conv2d(in_channels=8,\n",
|
||||||
|
" out_channels=8,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(2, 2),\n",
|
||||||
|
" padding=1) \n",
|
||||||
|
" \n",
|
||||||
|
" # 7x7x8 => 7x7x16 \n",
|
||||||
|
" self.conv_5 = torch.nn.Conv2d(in_channels=8,\n",
|
||||||
|
" out_channels=16,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(7-1) - 7 + 3) / 2 = 1 \n",
|
||||||
|
" # 7x7x16 => 4x4x16 \n",
|
||||||
|
" self.conv_6 = torch.nn.Conv2d(in_channels=16,\n",
|
||||||
|
" out_channels=16,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(2, 2),\n",
|
||||||
|
" padding=1) \n",
|
||||||
|
" \n",
|
||||||
|
" # 4x4x16 => 4x4xnum_classes \n",
|
||||||
|
" self.conv_7 = torch.nn.Conv2d(in_channels=16,\n",
|
||||||
|
" out_channels=self.num_classes,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(7-1) - 7 + 3) / 2 = 1 \n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" out = self.conv_1(x)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" \n",
|
||||||
|
" out = self.conv_2(out)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv_3(out)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv_4(out)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" \n",
|
||||||
|
" out = self.conv_5(out)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" \n",
|
||||||
|
" out = self.conv_6(out)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" \n",
|
||||||
|
" out = self.conv_7(out)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" \n",
|
||||||
|
" logits = F.adaptive_avg_pool2d(out, 1)\n",
|
||||||
|
" # drop width\n",
|
||||||
|
" logits.squeeze_(-1)\n",
|
||||||
|
" # drop height\n",
|
||||||
|
" logits.squeeze_(-1)\n",
|
||||||
|
" probas = torch.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"torch.manual_seed(random_seed)\n",
|
||||||
|
"model = ConvNet(num_classes=num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
"model = model.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/015 | Batch 000/235 | Cost: 2.3051\n",
|
||||||
|
"Epoch: 001/015 | Batch 050/235 | Cost: 2.2926\n",
|
||||||
|
"Epoch: 001/015 | Batch 100/235 | Cost: 2.0812\n",
|
||||||
|
"Epoch: 001/015 | Batch 150/235 | Cost: 1.4435\n",
|
||||||
|
"Epoch: 001/015 | Batch 200/235 | Cost: 0.9232\n",
|
||||||
|
"Epoch: 001/015 training accuracy: 76.06%\n",
|
||||||
|
"Time elapsed: 0.23 min\n",
|
||||||
|
"Epoch: 002/015 | Batch 000/235 | Cost: 0.7001\n",
|
||||||
|
"Epoch: 002/015 | Batch 050/235 | Cost: 0.5710\n",
|
||||||
|
"Epoch: 002/015 | Batch 100/235 | Cost: 0.5925\n",
|
||||||
|
"Epoch: 002/015 | Batch 150/235 | Cost: 0.4022\n",
|
||||||
|
"Epoch: 002/015 | Batch 200/235 | Cost: 0.4663\n",
|
||||||
|
"Epoch: 002/015 training accuracy: 85.68%\n",
|
||||||
|
"Time elapsed: 0.45 min\n",
|
||||||
|
"Epoch: 003/015 | Batch 000/235 | Cost: 0.4332\n",
|
||||||
|
"Epoch: 003/015 | Batch 050/235 | Cost: 0.3523\n",
|
||||||
|
"Epoch: 003/015 | Batch 100/235 | Cost: 0.4114\n",
|
||||||
|
"Epoch: 003/015 | Batch 150/235 | Cost: 0.4587\n",
|
||||||
|
"Epoch: 003/015 | Batch 200/235 | Cost: 0.4517\n",
|
||||||
|
"Epoch: 003/015 training accuracy: 89.33%\n",
|
||||||
|
"Time elapsed: 0.68 min\n",
|
||||||
|
"Epoch: 004/015 | Batch 000/235 | Cost: 0.4083\n",
|
||||||
|
"Epoch: 004/015 | Batch 050/235 | Cost: 0.3158\n",
|
||||||
|
"Epoch: 004/015 | Batch 100/235 | Cost: 0.2728\n",
|
||||||
|
"Epoch: 004/015 | Batch 150/235 | Cost: 0.3023\n",
|
||||||
|
"Epoch: 004/015 | Batch 200/235 | Cost: 0.2709\n",
|
||||||
|
"Epoch: 004/015 training accuracy: 90.40%\n",
|
||||||
|
"Time elapsed: 0.90 min\n",
|
||||||
|
"Epoch: 005/015 | Batch 000/235 | Cost: 0.2514\n",
|
||||||
|
"Epoch: 005/015 | Batch 050/235 | Cost: 0.3704\n",
|
||||||
|
"Epoch: 005/015 | Batch 100/235 | Cost: 0.2972\n",
|
||||||
|
"Epoch: 005/015 | Batch 150/235 | Cost: 0.2335\n",
|
||||||
|
"Epoch: 005/015 | Batch 200/235 | Cost: 0.3242\n",
|
||||||
|
"Epoch: 005/015 training accuracy: 91.36%\n",
|
||||||
|
"Time elapsed: 1.13 min\n",
|
||||||
|
"Epoch: 006/015 | Batch 000/235 | Cost: 0.3255\n",
|
||||||
|
"Epoch: 006/015 | Batch 050/235 | Cost: 0.2985\n",
|
||||||
|
"Epoch: 006/015 | Batch 100/235 | Cost: 0.3501\n",
|
||||||
|
"Epoch: 006/015 | Batch 150/235 | Cost: 0.2415\n",
|
||||||
|
"Epoch: 006/015 | Batch 200/235 | Cost: 0.1978\n",
|
||||||
|
"Epoch: 006/015 training accuracy: 92.82%\n",
|
||||||
|
"Time elapsed: 1.35 min\n",
|
||||||
|
"Epoch: 007/015 | Batch 000/235 | Cost: 0.1925\n",
|
||||||
|
"Epoch: 007/015 | Batch 050/235 | Cost: 0.2179\n",
|
||||||
|
"Epoch: 007/015 | Batch 100/235 | Cost: 0.3337\n",
|
||||||
|
"Epoch: 007/015 | Batch 150/235 | Cost: 0.1856\n",
|
||||||
|
"Epoch: 007/015 | Batch 200/235 | Cost: 0.1333\n",
|
||||||
|
"Epoch: 007/015 training accuracy: 93.68%\n",
|
||||||
|
"Time elapsed: 1.58 min\n",
|
||||||
|
"Epoch: 008/015 | Batch 000/235 | Cost: 0.1776\n",
|
||||||
|
"Epoch: 008/015 | Batch 050/235 | Cost: 0.2973\n",
|
||||||
|
"Epoch: 008/015 | Batch 100/235 | Cost: 0.1685\n",
|
||||||
|
"Epoch: 008/015 | Batch 150/235 | Cost: 0.2062\n",
|
||||||
|
"Epoch: 008/015 | Batch 200/235 | Cost: 0.2165\n",
|
||||||
|
"Epoch: 008/015 training accuracy: 94.42%\n",
|
||||||
|
"Time elapsed: 1.80 min\n",
|
||||||
|
"Epoch: 009/015 | Batch 000/235 | Cost: 0.2038\n",
|
||||||
|
"Epoch: 009/015 | Batch 050/235 | Cost: 0.1301\n",
|
||||||
|
"Epoch: 009/015 | Batch 100/235 | Cost: 0.1977\n",
|
||||||
|
"Epoch: 009/015 | Batch 150/235 | Cost: 0.2160\n",
|
||||||
|
"Epoch: 009/015 | Batch 200/235 | Cost: 0.1772\n",
|
||||||
|
"Epoch: 009/015 training accuracy: 94.61%\n",
|
||||||
|
"Time elapsed: 2.02 min\n",
|
||||||
|
"Epoch: 010/015 | Batch 000/235 | Cost: 0.1709\n",
|
||||||
|
"Epoch: 010/015 | Batch 050/235 | Cost: 0.1695\n",
|
||||||
|
"Epoch: 010/015 | Batch 100/235 | Cost: 0.2144\n",
|
||||||
|
"Epoch: 010/015 | Batch 150/235 | Cost: 0.1548\n",
|
||||||
|
"Epoch: 010/015 | Batch 200/235 | Cost: 0.1033\n",
|
||||||
|
"Epoch: 010/015 training accuracy: 94.90%\n",
|
||||||
|
"Time elapsed: 2.25 min\n",
|
||||||
|
"Epoch: 011/015 | Batch 000/235 | Cost: 0.1651\n",
|
||||||
|
"Epoch: 011/015 | Batch 050/235 | Cost: 0.1899\n",
|
||||||
|
"Epoch: 011/015 | Batch 100/235 | Cost: 0.1727\n",
|
||||||
|
"Epoch: 011/015 | Batch 150/235 | Cost: 0.1216\n",
|
||||||
|
"Epoch: 011/015 | Batch 200/235 | Cost: 0.1859\n",
|
||||||
|
"Epoch: 011/015 training accuracy: 94.82%\n",
|
||||||
|
"Time elapsed: 2.47 min\n",
|
||||||
|
"Epoch: 012/015 | Batch 000/235 | Cost: 0.2490\n",
|
||||||
|
"Epoch: 012/015 | Batch 050/235 | Cost: 0.1022\n",
|
||||||
|
"Epoch: 012/015 | Batch 100/235 | Cost: 0.0793\n",
|
||||||
|
"Epoch: 012/015 | Batch 150/235 | Cost: 0.2258\n",
|
||||||
|
"Epoch: 012/015 | Batch 200/235 | Cost: 0.1356\n",
|
||||||
|
"Epoch: 012/015 training accuracy: 95.35%\n",
|
||||||
|
"Time elapsed: 2.70 min\n",
|
||||||
|
"Epoch: 013/015 | Batch 000/235 | Cost: 0.1512\n",
|
||||||
|
"Epoch: 013/015 | Batch 050/235 | Cost: 0.1758\n",
|
||||||
|
"Epoch: 013/015 | Batch 100/235 | Cost: 0.1349\n",
|
||||||
|
"Epoch: 013/015 | Batch 150/235 | Cost: 0.1838\n",
|
||||||
|
"Epoch: 013/015 | Batch 200/235 | Cost: 0.1166\n",
|
||||||
|
"Epoch: 013/015 training accuracy: 95.61%\n",
|
||||||
|
"Time elapsed: 2.92 min\n",
|
||||||
|
"Epoch: 014/015 | Batch 000/235 | Cost: 0.1210\n",
|
||||||
|
"Epoch: 014/015 | Batch 050/235 | Cost: 0.1511\n",
|
||||||
|
"Epoch: 014/015 | Batch 100/235 | Cost: 0.1331\n",
|
||||||
|
"Epoch: 014/015 | Batch 150/235 | Cost: 0.1058\n",
|
||||||
|
"Epoch: 014/015 | Batch 200/235 | Cost: 0.1340\n",
|
||||||
|
"Epoch: 014/015 training accuracy: 95.53%\n",
|
||||||
|
"Time elapsed: 3.15 min\n",
|
||||||
|
"Epoch: 015/015 | Batch 000/235 | Cost: 0.2342\n",
|
||||||
|
"Epoch: 015/015 | Batch 050/235 | Cost: 0.1371\n",
|
||||||
|
"Epoch: 015/015 | Batch 100/235 | Cost: 0.0944\n",
|
||||||
|
"Epoch: 015/015 | Batch 150/235 | Cost: 0.1102\n",
|
||||||
|
"Epoch: 015/015 | Batch 200/235 | Cost: 0.1259\n",
|
||||||
|
"Epoch: 015/015 training accuracy: 96.36%\n",
|
||||||
|
"Time elapsed: 3.37 min\n",
|
||||||
|
"Total Training Time: 3.37 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for features, targets in data_loader:\n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(num_epochs):\n",
|
||||||
|
" model = model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, num_epochs, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
" \n",
|
||||||
|
" model = model.eval()\n",
|
||||||
|
" print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
|
||||||
|
" epoch+1, num_epochs, \n",
|
||||||
|
" compute_accuracy(model, train_loader)))\n",
|
||||||
|
" \n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 96.42%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"numpy 1.15.4\n",
|
||||||
|
"torch 1.0.1.post2\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.1"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": false,
|
||||||
|
"toc_position": {},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
@@ -0,0 +1,493 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.7.3\n",
|
||||||
|
"IPython 7.6.1\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.1.0\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Runs on CPU or GPU (if available)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- Convolutional Neural Network"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import time\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Settings and Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Device\n",
|
||||||
|
"device = torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"random_seed = 1\n",
|
||||||
|
"learning_rate = 0.05\n",
|
||||||
|
"num_epochs = 10\n",
|
||||||
|
"batch_size = 128\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"num_classes = 10\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Model"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class ConvNet(torch.nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, num_classes):\n",
|
||||||
|
" super(ConvNet, self).__init__()\n",
|
||||||
|
" \n",
|
||||||
|
" # calculate same padding:\n",
|
||||||
|
" # (w - k + 2*p)/s + 1 = o\n",
|
||||||
|
" # => p = (s(o-1) - w + k)/2\n",
|
||||||
|
" \n",
|
||||||
|
" # 28x28x1 => 28x28x8\n",
|
||||||
|
" self.conv_1 = torch.nn.Conv2d(in_channels=1,\n",
|
||||||
|
" out_channels=8,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(28-1) - 28 + 3) / 2 = 1\n",
|
||||||
|
" # 28x28x8 => 14x14x8\n",
|
||||||
|
" self.pool_1 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2),\n",
|
||||||
|
" padding=0) # (2(14-1) - 28 + 2) = 0 \n",
|
||||||
|
" # 14x14x8 => 14x14x16\n",
|
||||||
|
" self.conv_2 = torch.nn.Conv2d(in_channels=8,\n",
|
||||||
|
" out_channels=16,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(14-1) - 14 + 3) / 2 = 1 \n",
|
||||||
|
" # 14x14x16 => 7x7x16 \n",
|
||||||
|
" self.pool_2 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2),\n",
|
||||||
|
" padding=0) # (2(7-1) - 14 + 2) = 0\n",
|
||||||
|
"\n",
|
||||||
|
" self.linear_1 = torch.nn.Linear(7*7*16, num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
" # optionally initialize weights from Gaussian;\n",
|
||||||
|
" # Guassian weight init is not recommended and only for demonstration purposes\n",
|
||||||
|
" for m in self.modules():\n",
|
||||||
|
" if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):\n",
|
||||||
|
" m.weight.data.normal_(0.0, 0.01)\n",
|
||||||
|
" m.bias.data.zero_()\n",
|
||||||
|
" if m.bias is not None:\n",
|
||||||
|
" m.bias.detach().zero_()\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" out = self.conv_1(x)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" out = self.pool_1(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv_2(out)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" out = self.pool_2(out)\n",
|
||||||
|
" \n",
|
||||||
|
" logits = self.linear_1(out.view(-1, 7*7*16))\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"torch.manual_seed(random_seed)\n",
|
||||||
|
"model = ConvNet(num_classes=num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
"model = model.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 000/469 | Cost: 2.3026\n",
|
||||||
|
"Epoch: 001/010 | Batch 050/469 | Cost: 2.3036\n",
|
||||||
|
"Epoch: 001/010 | Batch 100/469 | Cost: 2.3001\n",
|
||||||
|
"Epoch: 001/010 | Batch 150/469 | Cost: 2.3050\n",
|
||||||
|
"Epoch: 001/010 | Batch 200/469 | Cost: 2.2984\n",
|
||||||
|
"Epoch: 001/010 | Batch 250/469 | Cost: 2.2986\n",
|
||||||
|
"Epoch: 001/010 | Batch 300/469 | Cost: 2.2983\n",
|
||||||
|
"Epoch: 001/010 | Batch 350/469 | Cost: 2.2941\n",
|
||||||
|
"Epoch: 001/010 | Batch 400/469 | Cost: 2.2962\n",
|
||||||
|
"Epoch: 001/010 | Batch 450/469 | Cost: 2.2265\n",
|
||||||
|
"Epoch: 001/010 training accuracy: 65.38%\n",
|
||||||
|
"Time elapsed: 0.24 min\n",
|
||||||
|
"Epoch: 002/010 | Batch 000/469 | Cost: 1.8989\n",
|
||||||
|
"Epoch: 002/010 | Batch 050/469 | Cost: 0.6029\n",
|
||||||
|
"Epoch: 002/010 | Batch 100/469 | Cost: 0.6099\n",
|
||||||
|
"Epoch: 002/010 | Batch 150/469 | Cost: 0.4786\n",
|
||||||
|
"Epoch: 002/010 | Batch 200/469 | Cost: 0.4518\n",
|
||||||
|
"Epoch: 002/010 | Batch 250/469 | Cost: 0.3553\n",
|
||||||
|
"Epoch: 002/010 | Batch 300/469 | Cost: 0.3167\n",
|
||||||
|
"Epoch: 002/010 | Batch 350/469 | Cost: 0.2241\n",
|
||||||
|
"Epoch: 002/010 | Batch 400/469 | Cost: 0.2259\n",
|
||||||
|
"Epoch: 002/010 | Batch 450/469 | Cost: 0.3056\n",
|
||||||
|
"Epoch: 002/010 training accuracy: 93.11%\n",
|
||||||
|
"Time elapsed: 0.47 min\n",
|
||||||
|
"Epoch: 003/010 | Batch 000/469 | Cost: 0.3313\n",
|
||||||
|
"Epoch: 003/010 | Batch 050/469 | Cost: 0.1042\n",
|
||||||
|
"Epoch: 003/010 | Batch 100/469 | Cost: 0.1328\n",
|
||||||
|
"Epoch: 003/010 | Batch 150/469 | Cost: 0.2803\n",
|
||||||
|
"Epoch: 003/010 | Batch 200/469 | Cost: 0.0975\n",
|
||||||
|
"Epoch: 003/010 | Batch 250/469 | Cost: 0.1839\n",
|
||||||
|
"Epoch: 003/010 | Batch 300/469 | Cost: 0.1774\n",
|
||||||
|
"Epoch: 003/010 | Batch 350/469 | Cost: 0.1143\n",
|
||||||
|
"Epoch: 003/010 | Batch 400/469 | Cost: 0.1753\n",
|
||||||
|
"Epoch: 003/010 | Batch 450/469 | Cost: 0.1543\n",
|
||||||
|
"Epoch: 003/010 training accuracy: 95.68%\n",
|
||||||
|
"Time elapsed: 0.70 min\n",
|
||||||
|
"Epoch: 004/010 | Batch 000/469 | Cost: 0.1057\n",
|
||||||
|
"Epoch: 004/010 | Batch 050/469 | Cost: 0.1035\n",
|
||||||
|
"Epoch: 004/010 | Batch 100/469 | Cost: 0.1851\n",
|
||||||
|
"Epoch: 004/010 | Batch 150/469 | Cost: 0.1608\n",
|
||||||
|
"Epoch: 004/010 | Batch 200/469 | Cost: 0.1458\n",
|
||||||
|
"Epoch: 004/010 | Batch 250/469 | Cost: 0.1913\n",
|
||||||
|
"Epoch: 004/010 | Batch 300/469 | Cost: 0.1295\n",
|
||||||
|
"Epoch: 004/010 | Batch 350/469 | Cost: 0.1518\n",
|
||||||
|
"Epoch: 004/010 | Batch 400/469 | Cost: 0.1717\n",
|
||||||
|
"Epoch: 004/010 | Batch 450/469 | Cost: 0.0792\n",
|
||||||
|
"Epoch: 004/010 training accuracy: 96.46%\n",
|
||||||
|
"Time elapsed: 0.93 min\n",
|
||||||
|
"Epoch: 005/010 | Batch 000/469 | Cost: 0.0905\n",
|
||||||
|
"Epoch: 005/010 | Batch 050/469 | Cost: 0.1622\n",
|
||||||
|
"Epoch: 005/010 | Batch 100/469 | Cost: 0.1934\n",
|
||||||
|
"Epoch: 005/010 | Batch 150/469 | Cost: 0.1874\n",
|
||||||
|
"Epoch: 005/010 | Batch 200/469 | Cost: 0.0742\n",
|
||||||
|
"Epoch: 005/010 | Batch 250/469 | Cost: 0.1056\n",
|
||||||
|
"Epoch: 005/010 | Batch 300/469 | Cost: 0.0997\n",
|
||||||
|
"Epoch: 005/010 | Batch 350/469 | Cost: 0.0948\n",
|
||||||
|
"Epoch: 005/010 | Batch 400/469 | Cost: 0.0575\n",
|
||||||
|
"Epoch: 005/010 | Batch 450/469 | Cost: 0.1157\n",
|
||||||
|
"Epoch: 005/010 training accuracy: 96.97%\n",
|
||||||
|
"Time elapsed: 1.16 min\n",
|
||||||
|
"Epoch: 006/010 | Batch 000/469 | Cost: 0.1326\n",
|
||||||
|
"Epoch: 006/010 | Batch 050/469 | Cost: 0.1549\n",
|
||||||
|
"Epoch: 006/010 | Batch 100/469 | Cost: 0.0784\n",
|
||||||
|
"Epoch: 006/010 | Batch 150/469 | Cost: 0.0898\n",
|
||||||
|
"Epoch: 006/010 | Batch 200/469 | Cost: 0.0991\n",
|
||||||
|
"Epoch: 006/010 | Batch 250/469 | Cost: 0.0965\n",
|
||||||
|
"Epoch: 006/010 | Batch 300/469 | Cost: 0.0477\n",
|
||||||
|
"Epoch: 006/010 | Batch 350/469 | Cost: 0.0712\n",
|
||||||
|
"Epoch: 006/010 | Batch 400/469 | Cost: 0.1109\n",
|
||||||
|
"Epoch: 006/010 | Batch 450/469 | Cost: 0.0325\n",
|
||||||
|
"Epoch: 006/010 training accuracy: 97.60%\n",
|
||||||
|
"Time elapsed: 1.39 min\n",
|
||||||
|
"Epoch: 007/010 | Batch 000/469 | Cost: 0.0665\n",
|
||||||
|
"Epoch: 007/010 | Batch 050/469 | Cost: 0.0868\n",
|
||||||
|
"Epoch: 007/010 | Batch 100/469 | Cost: 0.0427\n",
|
||||||
|
"Epoch: 007/010 | Batch 150/469 | Cost: 0.0385\n",
|
||||||
|
"Epoch: 007/010 | Batch 200/469 | Cost: 0.0611\n",
|
||||||
|
"Epoch: 007/010 | Batch 250/469 | Cost: 0.0484\n",
|
||||||
|
"Epoch: 007/010 | Batch 300/469 | Cost: 0.1288\n",
|
||||||
|
"Epoch: 007/010 | Batch 350/469 | Cost: 0.0309\n",
|
||||||
|
"Epoch: 007/010 | Batch 400/469 | Cost: 0.0359\n",
|
||||||
|
"Epoch: 007/010 | Batch 450/469 | Cost: 0.0139\n",
|
||||||
|
"Epoch: 007/010 training accuracy: 97.64%\n",
|
||||||
|
"Time elapsed: 1.62 min\n",
|
||||||
|
"Epoch: 008/010 | Batch 000/469 | Cost: 0.0939\n",
|
||||||
|
"Epoch: 008/010 | Batch 050/469 | Cost: 0.1478\n",
|
||||||
|
"Epoch: 008/010 | Batch 100/469 | Cost: 0.0769\n",
|
||||||
|
"Epoch: 008/010 | Batch 150/469 | Cost: 0.0713\n",
|
||||||
|
"Epoch: 008/010 | Batch 200/469 | Cost: 0.1272\n",
|
||||||
|
"Epoch: 008/010 | Batch 250/469 | Cost: 0.0446\n",
|
||||||
|
"Epoch: 008/010 | Batch 300/469 | Cost: 0.0525\n",
|
||||||
|
"Epoch: 008/010 | Batch 350/469 | Cost: 0.1729\n",
|
||||||
|
"Epoch: 008/010 | Batch 400/469 | Cost: 0.0672\n",
|
||||||
|
"Epoch: 008/010 | Batch 450/469 | Cost: 0.0754\n",
|
||||||
|
"Epoch: 008/010 training accuracy: 96.67%\n",
|
||||||
|
"Time elapsed: 1.85 min\n",
|
||||||
|
"Epoch: 009/010 | Batch 000/469 | Cost: 0.0988\n",
|
||||||
|
"Epoch: 009/010 | Batch 050/469 | Cost: 0.0409\n",
|
||||||
|
"Epoch: 009/010 | Batch 100/469 | Cost: 0.1046\n",
|
||||||
|
"Epoch: 009/010 | Batch 150/469 | Cost: 0.0523\n",
|
||||||
|
"Epoch: 009/010 | Batch 200/469 | Cost: 0.0815\n",
|
||||||
|
"Epoch: 009/010 | Batch 250/469 | Cost: 0.0811\n",
|
||||||
|
"Epoch: 009/010 | Batch 300/469 | Cost: 0.0416\n",
|
||||||
|
"Epoch: 009/010 | Batch 350/469 | Cost: 0.0747\n",
|
||||||
|
"Epoch: 009/010 | Batch 400/469 | Cost: 0.0467\n",
|
||||||
|
"Epoch: 009/010 | Batch 450/469 | Cost: 0.0669\n",
|
||||||
|
"Epoch: 009/010 training accuracy: 97.90%\n",
|
||||||
|
"Time elapsed: 2.08 min\n",
|
||||||
|
"Epoch: 010/010 | Batch 000/469 | Cost: 0.0257\n",
|
||||||
|
"Epoch: 010/010 | Batch 050/469 | Cost: 0.0357\n",
|
||||||
|
"Epoch: 010/010 | Batch 100/469 | Cost: 0.1469\n",
|
||||||
|
"Epoch: 010/010 | Batch 150/469 | Cost: 0.0170\n",
|
||||||
|
"Epoch: 010/010 | Batch 200/469 | Cost: 0.0493\n",
|
||||||
|
"Epoch: 010/010 | Batch 250/469 | Cost: 0.0489\n",
|
||||||
|
"Epoch: 010/010 | Batch 300/469 | Cost: 0.1348\n",
|
||||||
|
"Epoch: 010/010 | Batch 350/469 | Cost: 0.0815\n",
|
||||||
|
"Epoch: 010/010 | Batch 400/469 | Cost: 0.0552\n",
|
||||||
|
"Epoch: 010/010 | Batch 450/469 | Cost: 0.0422\n",
|
||||||
|
"Epoch: 010/010 training accuracy: 97.99%\n",
|
||||||
|
"Time elapsed: 2.31 min\n",
|
||||||
|
"Total Training Time: 2.31 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for features, targets in data_loader:\n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time() \n",
|
||||||
|
"for epoch in range(num_epochs):\n",
|
||||||
|
" model = model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, num_epochs, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
" \n",
|
||||||
|
" model = model.eval()\n",
|
||||||
|
" print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
|
||||||
|
" epoch+1, num_epochs, \n",
|
||||||
|
" compute_accuracy(model, train_loader)))\n",
|
||||||
|
"\n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 97.97%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"torch 1.1.0\n",
|
||||||
|
"numpy 1.16.4\n",
|
||||||
|
"torchvision 0.3.0\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.3"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": false,
|
||||||
|
"toc_position": {},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,214 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "4936d1e6-5e7d-4e22-ae35-8e888927ce2d",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Use Pre-trained CNN as feature extractor"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "bf9e9fb5-7383-475a-93e1-decdbd59c247",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Use MobileNetv3 as a feature extractor via the [embetter](https://github.com/koaning/embetter) scikit-learn library and [timm](https://github.com/rwightman/pytorch-image-models). Train a logistic regression classifier in scikit-learn on the embeddings."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "96b717c7-54c9-40dc-ba80-0fb47da2c0bd",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"id": "64d1dd64-c45b-4092-84d1-1bfcd0998f15",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"\n",
|
||||||
|
"# pip install gitpython\n",
|
||||||
|
"from git import Repo\n",
|
||||||
|
"\n",
|
||||||
|
"if not os.path.exists(\"mnist-pngs\"):\n",
|
||||||
|
" Repo.clone_from(\"https://github.com/rasbt/mnist-pngs\", \"mnist-pngs\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"id": "3a892538-8d9b-4420-9525-26d1a4b37ae3",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"\n",
|
||||||
|
"for name in (\"train\", \"test\"):\n",
|
||||||
|
"\n",
|
||||||
|
" df = pd.read_csv(f\"mnist-pngs/{name}.csv\")\n",
|
||||||
|
" df[\"filepath\"] = df[\"filepath\"].apply(lambda x: \"mnist-pngs/\" + x)\n",
|
||||||
|
" df = df.sample(frac=1, random_state=123).reset_index(drop=True)\n",
|
||||||
|
" df.to_csv(f\"mnist-pngs/{name}_shuffled.csv\", index=None)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"id": "5885e9bb-d43f-46ca-83ae-e2d63edcbb37",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "1fba0fcb2b1f408f85013da0d1694dd3",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
|
"text/plain": [
|
||||||
|
" 0%| | 0/60 [00:00<?, ?it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from sklearn.pipeline import make_pipeline\n",
|
||||||
|
"from sklearn.linear_model import SGDClassifier\n",
|
||||||
|
"from tqdm.notebook import tqdm\n",
|
||||||
|
"\n",
|
||||||
|
"# pip install \"embetter[vision]\"\n",
|
||||||
|
"from embetter.vision import ImageLoader, TimmEncoder\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"embed = make_pipeline(\n",
|
||||||
|
" ImageLoader(),\n",
|
||||||
|
" TimmEncoder(name=\"mobilenetv3_large_100\")\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"model = SGDClassifier(loss='log_loss', n_jobs=-1, shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"chunksize = 1000\n",
|
||||||
|
"train_labels, train_predict = [], []\n",
|
||||||
|
"\n",
|
||||||
|
"for df in tqdm(pd.read_csv(\"mnist-pngs/train_shuffled.csv\", chunksize=chunksize, iterator=True), total=60):\n",
|
||||||
|
" \n",
|
||||||
|
" embedded = embed.transform(df[\"filepath\"])\n",
|
||||||
|
" model.partial_fit(embedded, df[\"label\"], classes=list(range(10)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"id": "999a24ea-be5d-425f-923c-266372c66b5d",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "157302965ac8460c97c77935cc08e1fc",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
|
"text/plain": [
|
||||||
|
" 0%| | 0/60 [00:00<?, ?it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"train_labels, train_predict = [], []\n",
|
||||||
|
"\n",
|
||||||
|
"for df in tqdm(pd.read_csv(\"mnist-pngs/train.csv\", chunksize=chunksize, iterator=True), total=60):\n",
|
||||||
|
" df[\"filepath\"] = df[\"filepath\"].apply(lambda x: \"mnist-pngs/\" + x)\n",
|
||||||
|
"\n",
|
||||||
|
" embedded = embed.transform(df[\"filepath\"])\n",
|
||||||
|
" train_predict.extend(model.predict(embedded))\n",
|
||||||
|
" train_labels.extend(list(df[\"label\"].values))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"id": "c816cd7b-ed3a-4cb2-8aa6-400068a2e414",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "7869826407314279a2806bf602a796a8",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
|
"text/plain": [
|
||||||
|
" 0%| | 0/10 [00:00<?, ?it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"test_labels, test_predict = [], []\n",
|
||||||
|
"\n",
|
||||||
|
"for df in tqdm(pd.read_csv(\"mnist-pngs/test_shuffled.csv\", chunksize=chunksize, iterator=True), total=10):\n",
|
||||||
|
"\n",
|
||||||
|
" embedded = embed.transform(df[\"filepath\"])\n",
|
||||||
|
" test_predict.extend(model.predict(embedded))\n",
|
||||||
|
" test_labels.extend(list(df[\"label\"].values))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"id": "4a78add1-7f93-40fc-b119-9dbbe0aa55b4",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Train accuracy: 0.92\n",
|
||||||
|
"Test accuracy: 0.92\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from sklearn.metrics import accuracy_score\n",
|
||||||
|
"\n",
|
||||||
|
"print(f\"Train accuracy: {accuracy_score(train_labels, train_predict):.2f}\")\n",
|
||||||
|
"print(f\"Test accuracy: {accuracy_score(test_labels, test_predict):.2f}\")"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.9.7"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
||||||
@@ -0,0 +1,493 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.6.8\n",
|
||||||
|
"IPython 7.2.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.0.0\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Runs on CPU or GPU (if available)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- Convolutional Neural Network with He Initialization"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import time\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn as nn\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Settings and Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Device\n",
|
||||||
|
"device = torch.device(\"cuda:2\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"random_seed = 1\n",
|
||||||
|
"learning_rate = 0.05\n",
|
||||||
|
"num_epochs = 10\n",
|
||||||
|
"batch_size = 128\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"num_classes = 10\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Model"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class ConvNet(torch.nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, num_classes):\n",
|
||||||
|
" super(ConvNet, self).__init__()\n",
|
||||||
|
" \n",
|
||||||
|
" # calculate same padding:\n",
|
||||||
|
" # (w - k + 2*p)/s + 1 = o\n",
|
||||||
|
" # => p = (s(o-1) - w + k)/2\n",
|
||||||
|
" \n",
|
||||||
|
" # 28x28x1 => 28x28x4\n",
|
||||||
|
" self.conv_1 = torch.nn.Conv2d(in_channels=1,\n",
|
||||||
|
" out_channels=4,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(28-1) - 28 + 3) / 2 = 1\n",
|
||||||
|
" # 28x28x4 => 14x14x4\n",
|
||||||
|
" self.pool_1 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2),\n",
|
||||||
|
" padding=0) # (2(14-1) - 28 + 2) = 0 \n",
|
||||||
|
" # 14x14x4 => 14x14x8\n",
|
||||||
|
" self.conv_2 = torch.nn.Conv2d(in_channels=4,\n",
|
||||||
|
" out_channels=8,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(14-1) - 14 + 3) / 2 = 1 \n",
|
||||||
|
" # 14x14x8 => 7x7x8 \n",
|
||||||
|
" self.pool_2 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2),\n",
|
||||||
|
" padding=0) # (2(7-1) - 14 + 2) = 0\n",
|
||||||
|
" \n",
|
||||||
|
" self.linear_1 = torch.nn.Linear(7*7*8, num_classes)\n",
|
||||||
|
" \n",
|
||||||
|
" ###############################################\n",
|
||||||
|
" # Reinitialize weights using He initialization\n",
|
||||||
|
" ###############################################\n",
|
||||||
|
" for m in self.modules():\n",
|
||||||
|
" if isinstance(m, torch.nn.Conv2d):\n",
|
||||||
|
" nn.init.kaiming_normal_(m.weight.detach())\n",
|
||||||
|
" m.bias.detach().zero_()\n",
|
||||||
|
" elif isinstance(m, torch.nn.Linear):\n",
|
||||||
|
" nn.init.kaiming_normal_(m.weight.detach())\n",
|
||||||
|
" m.bias.detach().zero_()\n",
|
||||||
|
" \n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" out = self.conv_1(x)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" out = self.pool_1(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv_2(out)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" out = self.pool_2(out)\n",
|
||||||
|
" \n",
|
||||||
|
" logits = self.linear_1(out.view(-1, 7*7*8))\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"torch.manual_seed(random_seed)\n",
|
||||||
|
"model = ConvNet(num_classes=num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
"model = model.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 000/469 | Cost: 2.4577\n",
|
||||||
|
"Epoch: 001/010 | Batch 050/469 | Cost: 1.1068\n",
|
||||||
|
"Epoch: 001/010 | Batch 100/469 | Cost: 0.6610\n",
|
||||||
|
"Epoch: 001/010 | Batch 150/469 | Cost: 0.5354\n",
|
||||||
|
"Epoch: 001/010 | Batch 200/469 | Cost: 0.4479\n",
|
||||||
|
"Epoch: 001/010 | Batch 250/469 | Cost: 0.3158\n",
|
||||||
|
"Epoch: 001/010 | Batch 300/469 | Cost: 0.4542\n",
|
||||||
|
"Epoch: 001/010 | Batch 350/469 | Cost: 0.4278\n",
|
||||||
|
"Epoch: 001/010 | Batch 400/469 | Cost: 0.1387\n",
|
||||||
|
"Epoch: 001/010 | Batch 450/469 | Cost: 0.1410\n",
|
||||||
|
"Epoch: 001/010 training accuracy: 91.97%\n",
|
||||||
|
"Time elapsed: 0.23 min\n",
|
||||||
|
"Epoch: 002/010 | Batch 000/469 | Cost: 0.2198\n",
|
||||||
|
"Epoch: 002/010 | Batch 050/469 | Cost: 0.1464\n",
|
||||||
|
"Epoch: 002/010 | Batch 100/469 | Cost: 0.2629\n",
|
||||||
|
"Epoch: 002/010 | Batch 150/469 | Cost: 0.1920\n",
|
||||||
|
"Epoch: 002/010 | Batch 200/469 | Cost: 0.1485\n",
|
||||||
|
"Epoch: 002/010 | Batch 250/469 | Cost: 0.1229\n",
|
||||||
|
"Epoch: 002/010 | Batch 300/469 | Cost: 0.1591\n",
|
||||||
|
"Epoch: 002/010 | Batch 350/469 | Cost: 0.1411\n",
|
||||||
|
"Epoch: 002/010 | Batch 400/469 | Cost: 0.1404\n",
|
||||||
|
"Epoch: 002/010 | Batch 450/469 | Cost: 0.1211\n",
|
||||||
|
"Epoch: 002/010 training accuracy: 95.21%\n",
|
||||||
|
"Time elapsed: 0.46 min\n",
|
||||||
|
"Epoch: 003/010 | Batch 000/469 | Cost: 0.1289\n",
|
||||||
|
"Epoch: 003/010 | Batch 050/469 | Cost: 0.2468\n",
|
||||||
|
"Epoch: 003/010 | Batch 100/469 | Cost: 0.1308\n",
|
||||||
|
"Epoch: 003/010 | Batch 150/469 | Cost: 0.1887\n",
|
||||||
|
"Epoch: 003/010 | Batch 200/469 | Cost: 0.1053\n",
|
||||||
|
"Epoch: 003/010 | Batch 250/469 | Cost: 0.1564\n",
|
||||||
|
"Epoch: 003/010 | Batch 300/469 | Cost: 0.1235\n",
|
||||||
|
"Epoch: 003/010 | Batch 350/469 | Cost: 0.1388\n",
|
||||||
|
"Epoch: 003/010 | Batch 400/469 | Cost: 0.1556\n",
|
||||||
|
"Epoch: 003/010 | Batch 450/469 | Cost: 0.1658\n",
|
||||||
|
"Epoch: 003/010 training accuracy: 96.45%\n",
|
||||||
|
"Time elapsed: 0.69 min\n",
|
||||||
|
"Epoch: 004/010 | Batch 000/469 | Cost: 0.1827\n",
|
||||||
|
"Epoch: 004/010 | Batch 050/469 | Cost: 0.0613\n",
|
||||||
|
"Epoch: 004/010 | Batch 100/469 | Cost: 0.1967\n",
|
||||||
|
"Epoch: 004/010 | Batch 150/469 | Cost: 0.1072\n",
|
||||||
|
"Epoch: 004/010 | Batch 200/469 | Cost: 0.1063\n",
|
||||||
|
"Epoch: 004/010 | Batch 250/469 | Cost: 0.0970\n",
|
||||||
|
"Epoch: 004/010 | Batch 300/469 | Cost: 0.0593\n",
|
||||||
|
"Epoch: 004/010 | Batch 350/469 | Cost: 0.1031\n",
|
||||||
|
"Epoch: 004/010 | Batch 400/469 | Cost: 0.1503\n",
|
||||||
|
"Epoch: 004/010 | Batch 450/469 | Cost: 0.1611\n",
|
||||||
|
"Epoch: 004/010 training accuracy: 96.62%\n",
|
||||||
|
"Time elapsed: 0.92 min\n",
|
||||||
|
"Epoch: 005/010 | Batch 000/469 | Cost: 0.0469\n",
|
||||||
|
"Epoch: 005/010 | Batch 050/469 | Cost: 0.0351\n",
|
||||||
|
"Epoch: 005/010 | Batch 100/469 | Cost: 0.1232\n",
|
||||||
|
"Epoch: 005/010 | Batch 150/469 | Cost: 0.0432\n",
|
||||||
|
"Epoch: 005/010 | Batch 200/469 | Cost: 0.1049\n",
|
||||||
|
"Epoch: 005/010 | Batch 250/469 | Cost: 0.1136\n",
|
||||||
|
"Epoch: 005/010 | Batch 300/469 | Cost: 0.2226\n",
|
||||||
|
"Epoch: 005/010 | Batch 350/469 | Cost: 0.1271\n",
|
||||||
|
"Epoch: 005/010 | Batch 400/469 | Cost: 0.1405\n",
|
||||||
|
"Epoch: 005/010 | Batch 450/469 | Cost: 0.0651\n",
|
||||||
|
"Epoch: 005/010 training accuracy: 97.22%\n",
|
||||||
|
"Time elapsed: 1.15 min\n",
|
||||||
|
"Epoch: 006/010 | Batch 000/469 | Cost: 0.0886\n",
|
||||||
|
"Epoch: 006/010 | Batch 050/469 | Cost: 0.1358\n",
|
||||||
|
"Epoch: 006/010 | Batch 100/469 | Cost: 0.1083\n",
|
||||||
|
"Epoch: 006/010 | Batch 150/469 | Cost: 0.0799\n",
|
||||||
|
"Epoch: 006/010 | Batch 200/469 | Cost: 0.0815\n",
|
||||||
|
"Epoch: 006/010 | Batch 250/469 | Cost: 0.1873\n",
|
||||||
|
"Epoch: 006/010 | Batch 300/469 | Cost: 0.1785\n",
|
||||||
|
"Epoch: 006/010 | Batch 350/469 | Cost: 0.1107\n",
|
||||||
|
"Epoch: 006/010 | Batch 400/469 | Cost: 0.1059\n",
|
||||||
|
"Epoch: 006/010 | Batch 450/469 | Cost: 0.0741\n",
|
||||||
|
"Epoch: 006/010 training accuracy: 97.22%\n",
|
||||||
|
"Time elapsed: 1.38 min\n",
|
||||||
|
"Epoch: 007/010 | Batch 000/469 | Cost: 0.1303\n",
|
||||||
|
"Epoch: 007/010 | Batch 050/469 | Cost: 0.0944\n",
|
||||||
|
"Epoch: 007/010 | Batch 100/469 | Cost: 0.0867\n",
|
||||||
|
"Epoch: 007/010 | Batch 150/469 | Cost: 0.1706\n",
|
||||||
|
"Epoch: 007/010 | Batch 200/469 | Cost: 0.0840\n",
|
||||||
|
"Epoch: 007/010 | Batch 250/469 | Cost: 0.0876\n",
|
||||||
|
"Epoch: 007/010 | Batch 300/469 | Cost: 0.0565\n",
|
||||||
|
"Epoch: 007/010 | Batch 350/469 | Cost: 0.0805\n",
|
||||||
|
"Epoch: 007/010 | Batch 400/469 | Cost: 0.0784\n",
|
||||||
|
"Epoch: 007/010 | Batch 450/469 | Cost: 0.1238\n",
|
||||||
|
"Epoch: 007/010 training accuracy: 97.47%\n",
|
||||||
|
"Time elapsed: 1.62 min\n",
|
||||||
|
"Epoch: 008/010 | Batch 000/469 | Cost: 0.0740\n",
|
||||||
|
"Epoch: 008/010 | Batch 050/469 | Cost: 0.0674\n",
|
||||||
|
"Epoch: 008/010 | Batch 100/469 | Cost: 0.1884\n",
|
||||||
|
"Epoch: 008/010 | Batch 150/469 | Cost: 0.0757\n",
|
||||||
|
"Epoch: 008/010 | Batch 200/469 | Cost: 0.0633\n",
|
||||||
|
"Epoch: 008/010 | Batch 250/469 | Cost: 0.1166\n",
|
||||||
|
"Epoch: 008/010 | Batch 300/469 | Cost: 0.0309\n",
|
||||||
|
"Epoch: 008/010 | Batch 350/469 | Cost: 0.0821\n",
|
||||||
|
"Epoch: 008/010 | Batch 400/469 | Cost: 0.1253\n",
|
||||||
|
"Epoch: 008/010 | Batch 450/469 | Cost: 0.0486\n",
|
||||||
|
"Epoch: 008/010 training accuracy: 97.53%\n",
|
||||||
|
"Time elapsed: 1.85 min\n",
|
||||||
|
"Epoch: 009/010 | Batch 000/469 | Cost: 0.0538\n",
|
||||||
|
"Epoch: 009/010 | Batch 050/469 | Cost: 0.1860\n",
|
||||||
|
"Epoch: 009/010 | Batch 100/469 | Cost: 0.0645\n",
|
||||||
|
"Epoch: 009/010 | Batch 150/469 | Cost: 0.0392\n",
|
||||||
|
"Epoch: 009/010 | Batch 200/469 | Cost: 0.0662\n",
|
||||||
|
"Epoch: 009/010 | Batch 250/469 | Cost: 0.0885\n",
|
||||||
|
"Epoch: 009/010 | Batch 300/469 | Cost: 0.1958\n",
|
||||||
|
"Epoch: 009/010 | Batch 350/469 | Cost: 0.0716\n",
|
||||||
|
"Epoch: 009/010 | Batch 400/469 | Cost: 0.0790\n",
|
||||||
|
"Epoch: 009/010 | Batch 450/469 | Cost: 0.0223\n",
|
||||||
|
"Epoch: 009/010 training accuracy: 97.89%\n",
|
||||||
|
"Time elapsed: 2.08 min\n",
|
||||||
|
"Epoch: 010/010 | Batch 000/469 | Cost: 0.0982\n",
|
||||||
|
"Epoch: 010/010 | Batch 050/469 | Cost: 0.0772\n",
|
||||||
|
"Epoch: 010/010 | Batch 100/469 | Cost: 0.1971\n",
|
||||||
|
"Epoch: 010/010 | Batch 150/469 | Cost: 0.0399\n",
|
||||||
|
"Epoch: 010/010 | Batch 200/469 | Cost: 0.0341\n",
|
||||||
|
"Epoch: 010/010 | Batch 250/469 | Cost: 0.0538\n",
|
||||||
|
"Epoch: 010/010 | Batch 300/469 | Cost: 0.1165\n",
|
||||||
|
"Epoch: 010/010 | Batch 350/469 | Cost: 0.1016\n",
|
||||||
|
"Epoch: 010/010 | Batch 400/469 | Cost: 0.1560\n",
|
||||||
|
"Epoch: 010/010 | Batch 450/469 | Cost: 0.1757\n",
|
||||||
|
"Epoch: 010/010 training accuracy: 97.80%\n",
|
||||||
|
"Time elapsed: 2.31 min\n",
|
||||||
|
"Total Training Time: 2.31 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for features, targets in data_loader:\n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(num_epochs):\n",
|
||||||
|
" model = model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, num_epochs, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
" \n",
|
||||||
|
" model = model.eval()\n",
|
||||||
|
" print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
|
||||||
|
" epoch+1, num_epochs, \n",
|
||||||
|
" compute_accuracy(model, train_loader)))\n",
|
||||||
|
" \n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 97.67%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"numpy 1.15.4\n",
|
||||||
|
"torch 1.0.0\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.3"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": false,
|
||||||
|
"toc_position": {},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
@@ -0,0 +1,696 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "UEBilEjLj5wY"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 119
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 536,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974472601,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "GOzuY8Yvj5wb",
|
||||||
|
"outputId": "c19362ce-f87a-4cc2-84cc-8d7b4b9e6007"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.7.3\n",
|
||||||
|
"IPython 7.9.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.3.1\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "rH4XmErYj5wm"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"# LeNet-5 CIFAR10 Classifier"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"This notebook implements the classic LeNet-5 convolutional network [1] and applies it to the CIFAR10 object classification dataset. The basic architecture is shown in the figure below:\n",
|
||||||
|
"\n",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"LeNet-5 is commonly regarded as the pioneer of convolutional neural networks, consisting of a very simple architecture (by modern standards). In total, LeNet-5 consists of only 7 layers. 3 out of these 7 layers are convolutional layers (C1, C3, C5), which are connected by two average pooling layers (S2 & S4). The penultimate layer is a fully connexted layer (F6), which is followed by the final output layer. The additional details are summarized below:\n",
|
||||||
|
"\n",
|
||||||
|
"- All convolutional layers use 5x5 kernels with stride 1.\n",
|
||||||
|
"- The two average pooling (subsampling) layers are 2x2 pixels wide with stride 1.\n",
|
||||||
|
"- Throughrout the network, tanh sigmoid activation functions are used. (**In this notebook, we replace these with ReLU activations**)\n",
|
||||||
|
"- The output layer uses 10 custom Euclidean Radial Basis Function neurons for the output layer. (**In this notebook, we replace these with softmax activations**)\n",
|
||||||
|
"\n",
|
||||||
|
"**Please note that the original architecture was applied to MNIST-like grayscale images (1 color channel). CIFAR10 has 3 color-channels. I found that using the regular architecture results in very poor performance on CIFAR10 (approx. 50% ACC). Hence, I am multiplying the number of kernels by a factor of 3 (according to the 3 color channels) in each layer, which improves is a little bit (approx. 60% Acc).**\n",
|
||||||
|
"\n",
|
||||||
|
"### References\n",
|
||||||
|
"\n",
|
||||||
|
"- [1] Y. LeCun, L. Bottou, Y. Bengio, and P. Haffner. Gradient-based learning applied to document recognition. Proceedings of the IEEE, november 1998."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MkoGLH_Tj5wn"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "ORj09gnrj5wp"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import time\n",
|
||||||
|
"\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn as nn\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"from PIL import Image\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "I6hghKPxj5w0"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Model Settings"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 85
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 23936,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974497505,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "NnT0sZIwj5wu",
|
||||||
|
"outputId": "55aed925-d17e-4c6a-8c71-0d9b3bde5637"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"RANDOM_SEED = 1\n",
|
||||||
|
"LEARNING_RATE = 0.001\n",
|
||||||
|
"BATCH_SIZE = 128\n",
|
||||||
|
"NUM_EPOCHS = 10\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"NUM_FEATURES = 32*32\n",
|
||||||
|
"NUM_CLASSES = 10\n",
|
||||||
|
"\n",
|
||||||
|
"# Other\n",
|
||||||
|
"DEVICE = \"cuda:0\"\n",
|
||||||
|
"GRAYSCALE = False"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### MNIST Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Files already downloaded and verified\n",
|
||||||
|
"Image batch dimensions: torch.Size([128, 3, 32, 32])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n",
|
||||||
|
"Image batch dimensions: torch.Size([128, 3, 32, 32])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### CIFAR-10 Dataset\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE, \n",
|
||||||
|
" num_workers=8,\n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE,\n",
|
||||||
|
" num_workers=8,\n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 1 | Batch index: 0 | Batch size: 128\n",
|
||||||
|
"Epoch: 2 | Batch index: 0 | Batch size: 128\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"device = torch.device(DEVICE)\n",
|
||||||
|
"torch.manual_seed(0)\n",
|
||||||
|
"\n",
|
||||||
|
"for epoch in range(2):\n",
|
||||||
|
"\n",
|
||||||
|
" for batch_idx, (x, y) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" print('Epoch:', epoch+1, end='')\n",
|
||||||
|
" print(' | Batch index:', batch_idx, end='')\n",
|
||||||
|
" print(' | Batch size:', y.size()[0])\n",
|
||||||
|
" \n",
|
||||||
|
" x = x.to(device)\n",
|
||||||
|
" y = y.to(device)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class LeNet5(nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, num_classes, grayscale=False):\n",
|
||||||
|
" super(LeNet5, self).__init__()\n",
|
||||||
|
" \n",
|
||||||
|
" self.grayscale = grayscale\n",
|
||||||
|
" self.num_classes = num_classes\n",
|
||||||
|
"\n",
|
||||||
|
" if self.grayscale:\n",
|
||||||
|
" in_channels = 1\n",
|
||||||
|
" else:\n",
|
||||||
|
" in_channels = 3\n",
|
||||||
|
"\n",
|
||||||
|
" self.features = nn.Sequential(\n",
|
||||||
|
" \n",
|
||||||
|
" nn.Conv2d(in_channels, 6*in_channels, kernel_size=5),\n",
|
||||||
|
" nn.Tanh(),\n",
|
||||||
|
" nn.MaxPool2d(kernel_size=2),\n",
|
||||||
|
" nn.Conv2d(6*in_channels, 16*in_channels, kernel_size=5),\n",
|
||||||
|
" nn.Tanh(),\n",
|
||||||
|
" nn.MaxPool2d(kernel_size=2)\n",
|
||||||
|
" )\n",
|
||||||
|
"\n",
|
||||||
|
" self.classifier = nn.Sequential(\n",
|
||||||
|
" nn.Linear(16*5*5*in_channels, 120*in_channels),\n",
|
||||||
|
" nn.Tanh(),\n",
|
||||||
|
" nn.Linear(120*in_channels, 84*in_channels),\n",
|
||||||
|
" nn.Tanh(),\n",
|
||||||
|
" nn.Linear(84*in_channels, num_classes),\n",
|
||||||
|
" )\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" x = self.features(x)\n",
|
||||||
|
" x = torch.flatten(x, 1)\n",
|
||||||
|
" logits = self.classifier(x)\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "_lza9t_uj5w1"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"torch.manual_seed(RANDOM_SEED)\n",
|
||||||
|
"\n",
|
||||||
|
"model = LeNet5(NUM_CLASSES, GRAYSCALE)\n",
|
||||||
|
"model.to(DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "RAodboScj5w6"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 1547
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 2384585,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976888520,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "Dzh3ROmRj5w7",
|
||||||
|
"outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 0000/0391 | Cost: 2.3068\n",
|
||||||
|
"Epoch: 001/010 | Batch 0050/0391 | Cost: 1.8193\n",
|
||||||
|
"Epoch: 001/010 | Batch 0100/0391 | Cost: 1.6464\n",
|
||||||
|
"Epoch: 001/010 | Batch 0150/0391 | Cost: 1.5757\n",
|
||||||
|
"Epoch: 001/010 | Batch 0200/0391 | Cost: 1.4026\n",
|
||||||
|
"Epoch: 001/010 | Batch 0250/0391 | Cost: 1.3116\n",
|
||||||
|
"Epoch: 001/010 | Batch 0300/0391 | Cost: 1.3310\n",
|
||||||
|
"Epoch: 001/010 | Batch 0350/0391 | Cost: 1.2781\n",
|
||||||
|
"Epoch: 001/010 | Train: 54.326%\n",
|
||||||
|
"Time elapsed: 0.16 min\n",
|
||||||
|
"Epoch: 002/010 | Batch 0000/0391 | Cost: 1.4109\n",
|
||||||
|
"Epoch: 002/010 | Batch 0050/0391 | Cost: 1.3039\n",
|
||||||
|
"Epoch: 002/010 | Batch 0100/0391 | Cost: 1.2601\n",
|
||||||
|
"Epoch: 002/010 | Batch 0150/0391 | Cost: 1.3187\n",
|
||||||
|
"Epoch: 002/010 | Batch 0200/0391 | Cost: 1.2844\n",
|
||||||
|
"Epoch: 002/010 | Batch 0250/0391 | Cost: 1.3451\n",
|
||||||
|
"Epoch: 002/010 | Batch 0300/0391 | Cost: 1.1971\n",
|
||||||
|
"Epoch: 002/010 | Batch 0350/0391 | Cost: 1.1474\n",
|
||||||
|
"Epoch: 002/010 | Train: 60.528%\n",
|
||||||
|
"Time elapsed: 0.31 min\n",
|
||||||
|
"Epoch: 003/010 | Batch 0000/0391 | Cost: 1.1268\n",
|
||||||
|
"Epoch: 003/010 | Batch 0050/0391 | Cost: 1.1943\n",
|
||||||
|
"Epoch: 003/010 | Batch 0100/0391 | Cost: 1.3056\n",
|
||||||
|
"Epoch: 003/010 | Batch 0150/0391 | Cost: 1.0215\n",
|
||||||
|
"Epoch: 003/010 | Batch 0200/0391 | Cost: 1.0243\n",
|
||||||
|
"Epoch: 003/010 | Batch 0250/0391 | Cost: 0.7985\n",
|
||||||
|
"Epoch: 003/010 | Batch 0300/0391 | Cost: 1.0755\n",
|
||||||
|
"Epoch: 003/010 | Batch 0350/0391 | Cost: 1.1030\n",
|
||||||
|
"Epoch: 003/010 | Train: 64.586%\n",
|
||||||
|
"Time elapsed: 0.46 min\n",
|
||||||
|
"Epoch: 004/010 | Batch 0000/0391 | Cost: 1.1329\n",
|
||||||
|
"Epoch: 004/010 | Batch 0050/0391 | Cost: 1.0834\n",
|
||||||
|
"Epoch: 004/010 | Batch 0100/0391 | Cost: 1.0509\n",
|
||||||
|
"Epoch: 004/010 | Batch 0150/0391 | Cost: 0.9873\n",
|
||||||
|
"Epoch: 004/010 | Batch 0200/0391 | Cost: 0.8560\n",
|
||||||
|
"Epoch: 004/010 | Batch 0250/0391 | Cost: 1.1286\n",
|
||||||
|
"Epoch: 004/010 | Batch 0300/0391 | Cost: 0.8377\n",
|
||||||
|
"Epoch: 004/010 | Batch 0350/0391 | Cost: 1.1735\n",
|
||||||
|
"Epoch: 004/010 | Train: 66.656%\n",
|
||||||
|
"Time elapsed: 0.61 min\n",
|
||||||
|
"Epoch: 005/010 | Batch 0000/0391 | Cost: 1.1260\n",
|
||||||
|
"Epoch: 005/010 | Batch 0050/0391 | Cost: 0.8605\n",
|
||||||
|
"Epoch: 005/010 | Batch 0100/0391 | Cost: 0.9007\n",
|
||||||
|
"Epoch: 005/010 | Batch 0150/0391 | Cost: 0.9166\n",
|
||||||
|
"Epoch: 005/010 | Batch 0200/0391 | Cost: 0.9488\n",
|
||||||
|
"Epoch: 005/010 | Batch 0250/0391 | Cost: 1.0388\n",
|
||||||
|
"Epoch: 005/010 | Batch 0300/0391 | Cost: 0.9526\n",
|
||||||
|
"Epoch: 005/010 | Batch 0350/0391 | Cost: 0.9109\n",
|
||||||
|
"Epoch: 005/010 | Train: 71.504%\n",
|
||||||
|
"Time elapsed: 0.76 min\n",
|
||||||
|
"Epoch: 006/010 | Batch 0000/0391 | Cost: 0.7038\n",
|
||||||
|
"Epoch: 006/010 | Batch 0050/0391 | Cost: 0.6849\n",
|
||||||
|
"Epoch: 006/010 | Batch 0100/0391 | Cost: 0.6817\n",
|
||||||
|
"Epoch: 006/010 | Batch 0150/0391 | Cost: 0.8213\n",
|
||||||
|
"Epoch: 006/010 | Batch 0200/0391 | Cost: 0.7984\n",
|
||||||
|
"Epoch: 006/010 | Batch 0250/0391 | Cost: 0.9680\n",
|
||||||
|
"Epoch: 006/010 | Batch 0300/0391 | Cost: 0.7650\n",
|
||||||
|
"Epoch: 006/010 | Batch 0350/0391 | Cost: 0.9355\n",
|
||||||
|
"Epoch: 006/010 | Train: 74.812%\n",
|
||||||
|
"Time elapsed: 0.91 min\n",
|
||||||
|
"Epoch: 007/010 | Batch 0000/0391 | Cost: 0.8488\n",
|
||||||
|
"Epoch: 007/010 | Batch 0050/0391 | Cost: 0.8332\n",
|
||||||
|
"Epoch: 007/010 | Batch 0100/0391 | Cost: 0.6777\n",
|
||||||
|
"Epoch: 007/010 | Batch 0150/0391 | Cost: 0.6288\n",
|
||||||
|
"Epoch: 007/010 | Batch 0200/0391 | Cost: 0.6278\n",
|
||||||
|
"Epoch: 007/010 | Batch 0250/0391 | Cost: 0.6197\n",
|
||||||
|
"Epoch: 007/010 | Batch 0300/0391 | Cost: 0.7163\n",
|
||||||
|
"Epoch: 007/010 | Batch 0350/0391 | Cost: 0.7765\n",
|
||||||
|
"Epoch: 007/010 | Train: 78.272%\n",
|
||||||
|
"Time elapsed: 1.06 min\n",
|
||||||
|
"Epoch: 008/010 | Batch 0000/0391 | Cost: 0.5051\n",
|
||||||
|
"Epoch: 008/010 | Batch 0050/0391 | Cost: 0.5975\n",
|
||||||
|
"Epoch: 008/010 | Batch 0100/0391 | Cost: 0.6060\n",
|
||||||
|
"Epoch: 008/010 | Batch 0150/0391 | Cost: 0.6763\n",
|
||||||
|
"Epoch: 008/010 | Batch 0200/0391 | Cost: 0.5805\n",
|
||||||
|
"Epoch: 008/010 | Batch 0250/0391 | Cost: 0.6076\n",
|
||||||
|
"Epoch: 008/010 | Batch 0300/0391 | Cost: 0.5982\n",
|
||||||
|
"Epoch: 008/010 | Batch 0350/0391 | Cost: 0.8050\n",
|
||||||
|
"Epoch: 008/010 | Train: 82.530%\n",
|
||||||
|
"Time elapsed: 1.22 min\n",
|
||||||
|
"Epoch: 009/010 | Batch 0000/0391 | Cost: 0.4763\n",
|
||||||
|
"Epoch: 009/010 | Batch 0050/0391 | Cost: 0.4632\n",
|
||||||
|
"Epoch: 009/010 | Batch 0100/0391 | Cost: 0.6612\n",
|
||||||
|
"Epoch: 009/010 | Batch 0150/0391 | Cost: 0.5145\n",
|
||||||
|
"Epoch: 009/010 | Batch 0200/0391 | Cost: 0.6276\n",
|
||||||
|
"Epoch: 009/010 | Batch 0250/0391 | Cost: 0.7371\n",
|
||||||
|
"Epoch: 009/010 | Batch 0300/0391 | Cost: 0.6105\n",
|
||||||
|
"Epoch: 009/010 | Batch 0350/0391 | Cost: 0.6129\n",
|
||||||
|
"Epoch: 009/010 | Train: 84.632%\n",
|
||||||
|
"Time elapsed: 1.37 min\n",
|
||||||
|
"Epoch: 010/010 | Batch 0000/0391 | Cost: 0.4477\n",
|
||||||
|
"Epoch: 010/010 | Batch 0050/0391 | Cost: 0.3956\n",
|
||||||
|
"Epoch: 010/010 | Batch 0100/0391 | Cost: 0.4634\n",
|
||||||
|
"Epoch: 010/010 | Batch 0150/0391 | Cost: 0.4358\n",
|
||||||
|
"Epoch: 010/010 | Batch 0200/0391 | Cost: 0.5757\n",
|
||||||
|
"Epoch: 010/010 | Batch 0250/0391 | Cost: 0.4659\n",
|
||||||
|
"Epoch: 010/010 | Batch 0300/0391 | Cost: 0.4999\n",
|
||||||
|
"Epoch: 010/010 | Batch 0350/0391 | Cost: 0.4897\n",
|
||||||
|
"Epoch: 010/010 | Train: 88.534%\n",
|
||||||
|
"Time elapsed: 1.51 min\n",
|
||||||
|
"Total Training Time: 1.51 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader, device):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for i, (features, targets) in enumerate(data_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(NUM_EPOCHS):\n",
|
||||||
|
" \n",
|
||||||
|
" model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" \n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, NUM_EPOCHS, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Epoch: %03d/%03d | Train: %.3f%%' % (\n",
|
||||||
|
" epoch+1, NUM_EPOCHS, \n",
|
||||||
|
" compute_accuracy(model, train_loader, device=DEVICE)))\n",
|
||||||
|
" \n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "paaeEQHQj5xC"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 34
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 6514,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976895054,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "gzQMWKq5j5xE",
|
||||||
|
"outputId": "de7dc005-5eeb-4177-9f9f-d9b5d1358db9"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 67.30%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader, device=DEVICE)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"torch 1.3.1\n",
|
||||||
|
"pandas 0.24.2\n",
|
||||||
|
"PIL.Image 6.2.1\n",
|
||||||
|
"torchvision 0.4.2\n",
|
||||||
|
"matplotlib 3.1.0\n",
|
||||||
|
"numpy 1.17.4\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"accelerator": "GPU",
|
||||||
|
"colab": {
|
||||||
|
"collapsed_sections": [],
|
||||||
|
"default_view": {},
|
||||||
|
"name": "convnet-vgg16.ipynb",
|
||||||
|
"provenance": [],
|
||||||
|
"version": "0.3.2",
|
||||||
|
"views": {}
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.3"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": true,
|
||||||
|
"toc_position": {
|
||||||
|
"height": "calc(100% - 180px)",
|
||||||
|
"left": "10px",
|
||||||
|
"top": "150px",
|
||||||
|
"width": "371px"
|
||||||
|
},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,849 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "UEBilEjLj5wY"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 119
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 536,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974472601,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "GOzuY8Yvj5wb",
|
||||||
|
"outputId": "c19362ce-f87a-4cc2-84cc-8d7b4b9e6007"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.6.8\n",
|
||||||
|
"IPython 7.2.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.0.0\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "rH4XmErYj5wm"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- ResNet-18 MNIST Digits Classifier with Data Parallelism"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Network Architecture"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The network in this notebook is an implementation of the ResNet-18 [1] architecture on the MNIST digits dataset (http://yann.lecun.com/exdb/mnist/) to train a handwritten digit classifier. \n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"References\n",
|
||||||
|
" \n",
|
||||||
|
"- [1] He, K., Zhang, X., Ren, S., & Sun, J. (2016). Deep residual learning for image recognition. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 770-778). ([CVPR Link](https://www.cv-foundation.org/openaccess/content_cvpr_2016/html/He_Deep_Residual_Learning_CVPR_2016_paper.html))\n",
|
||||||
|
"\n",
|
||||||
|
"- [2] http://yann.lecun.com/exdb/mnist/"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The following figure illustrates residual blocks with skip connections such that the input passed via the shortcut matches the dimensions of the main path's output, which allows the network to learn identity functions.\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"The ResNet-18 architecture actually uses residual blocks with skip connections such that the input passed via the shortcut matches is resized to dimensions of the main path's output. Such a residual block is illustrated below:\n",
|
||||||
|
"\n",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"For a more detailed explanation see the other notebook, [resnet-ex-1.ipynb](resnet-ex-1.ipynb)."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MkoGLH_Tj5wn"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "ORj09gnrj5wp"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import time\n",
|
||||||
|
"\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn as nn\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"from PIL import Image\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "I6hghKPxj5w0"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Model Settings"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 85
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 23936,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974497505,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "NnT0sZIwj5wu",
|
||||||
|
"outputId": "55aed925-d17e-4c6a-8c71-0d9b3bde5637"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"RANDOM_SEED = 1\n",
|
||||||
|
"LEARNING_RATE = 0.001\n",
|
||||||
|
"BATCH_SIZE = 128\n",
|
||||||
|
"NUM_EPOCHS = 10\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"NUM_FEATURES = 28*28\n",
|
||||||
|
"NUM_CLASSES = 10\n",
|
||||||
|
"\n",
|
||||||
|
"# Other\n",
|
||||||
|
"DEVICE = \"cuda:1\"\n",
|
||||||
|
"GRAYSCALE = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### MNIST Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 1 | Batch index: 0 | Batch size: 128\n",
|
||||||
|
"Epoch: 2 | Batch index: 0 | Batch size: 128\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"device = torch.device(DEVICE)\n",
|
||||||
|
"torch.manual_seed(0)\n",
|
||||||
|
"\n",
|
||||||
|
"for epoch in range(2):\n",
|
||||||
|
"\n",
|
||||||
|
" for batch_idx, (x, y) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" print('Epoch:', epoch+1, end='')\n",
|
||||||
|
" print(' | Batch index:', batch_idx, end='')\n",
|
||||||
|
" print(' | Batch size:', y.size()[0])\n",
|
||||||
|
" \n",
|
||||||
|
" x = x.to(device)\n",
|
||||||
|
" y = y.to(device)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The following code cell that implements the ResNet-34 architecture is a derivative of the code provided at https://pytorch.org/docs/0.4.0/_modules/torchvision/models/resnet.html."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def conv3x3(in_planes, out_planes, stride=1):\n",
|
||||||
|
" \"\"\"3x3 convolution with padding\"\"\"\n",
|
||||||
|
" return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
|
||||||
|
" padding=1, bias=False)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class BasicBlock(nn.Module):\n",
|
||||||
|
" expansion = 1\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, inplanes, planes, stride=1, downsample=None):\n",
|
||||||
|
" super(BasicBlock, self).__init__()\n",
|
||||||
|
" self.conv1 = conv3x3(inplanes, planes, stride)\n",
|
||||||
|
" self.bn1 = nn.BatchNorm2d(planes)\n",
|
||||||
|
" self.relu = nn.ReLU(inplace=True)\n",
|
||||||
|
" self.conv2 = conv3x3(planes, planes)\n",
|
||||||
|
" self.bn2 = nn.BatchNorm2d(planes)\n",
|
||||||
|
" self.downsample = downsample\n",
|
||||||
|
" self.stride = stride\n",
|
||||||
|
"\n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" residual = x\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv1(x)\n",
|
||||||
|
" out = self.bn1(out)\n",
|
||||||
|
" out = self.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv2(out)\n",
|
||||||
|
" out = self.bn2(out)\n",
|
||||||
|
"\n",
|
||||||
|
" if self.downsample is not None:\n",
|
||||||
|
" residual = self.downsample(x)\n",
|
||||||
|
"\n",
|
||||||
|
" out += residual\n",
|
||||||
|
" out = self.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" return out\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class ResNet(nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, block, layers, num_classes, grayscale):\n",
|
||||||
|
" self.inplanes = 64\n",
|
||||||
|
" if grayscale:\n",
|
||||||
|
" in_dim = 1\n",
|
||||||
|
" else:\n",
|
||||||
|
" in_dim = 3\n",
|
||||||
|
" super(ResNet, self).__init__()\n",
|
||||||
|
" self.conv1 = nn.Conv2d(in_dim, 64, kernel_size=7, stride=2, padding=3,\n",
|
||||||
|
" bias=False)\n",
|
||||||
|
" self.bn1 = nn.BatchNorm2d(64)\n",
|
||||||
|
" self.relu = nn.ReLU(inplace=True)\n",
|
||||||
|
" self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
|
||||||
|
" self.layer1 = self._make_layer(block, 64, layers[0])\n",
|
||||||
|
" self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n",
|
||||||
|
" self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n",
|
||||||
|
" self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n",
|
||||||
|
" self.avgpool = nn.AvgPool2d(7, stride=1)\n",
|
||||||
|
" self.fc = nn.Linear(512 * block.expansion, num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
" for m in self.modules():\n",
|
||||||
|
" if isinstance(m, nn.Conv2d):\n",
|
||||||
|
" n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
|
||||||
|
" m.weight.data.normal_(0, (2. / n)**.5)\n",
|
||||||
|
" elif isinstance(m, nn.BatchNorm2d):\n",
|
||||||
|
" m.weight.data.fill_(1)\n",
|
||||||
|
" m.bias.data.zero_()\n",
|
||||||
|
"\n",
|
||||||
|
" def _make_layer(self, block, planes, blocks, stride=1):\n",
|
||||||
|
" downsample = None\n",
|
||||||
|
" if stride != 1 or self.inplanes != planes * block.expansion:\n",
|
||||||
|
" downsample = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(self.inplanes, planes * block.expansion,\n",
|
||||||
|
" kernel_size=1, stride=stride, bias=False),\n",
|
||||||
|
" nn.BatchNorm2d(planes * block.expansion),\n",
|
||||||
|
" )\n",
|
||||||
|
"\n",
|
||||||
|
" layers = []\n",
|
||||||
|
" layers.append(block(self.inplanes, planes, stride, downsample))\n",
|
||||||
|
" self.inplanes = planes * block.expansion\n",
|
||||||
|
" for i in range(1, blocks):\n",
|
||||||
|
" layers.append(block(self.inplanes, planes))\n",
|
||||||
|
"\n",
|
||||||
|
" return nn.Sequential(*layers)\n",
|
||||||
|
"\n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" x = self.conv1(x)\n",
|
||||||
|
" x = self.bn1(x)\n",
|
||||||
|
" x = self.relu(x)\n",
|
||||||
|
" x = self.maxpool(x)\n",
|
||||||
|
"\n",
|
||||||
|
" x = self.layer1(x)\n",
|
||||||
|
" x = self.layer2(x)\n",
|
||||||
|
" x = self.layer3(x)\n",
|
||||||
|
" x = self.layer4(x)\n",
|
||||||
|
" # because MNIST is already 1x1 here:\n",
|
||||||
|
" # disable avg pooling\n",
|
||||||
|
" #x = self.avgpool(x)\n",
|
||||||
|
" \n",
|
||||||
|
" x = x.view(x.size(0), -1)\n",
|
||||||
|
" logits = self.fc(x)\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def resnet18(num_classes):\n",
|
||||||
|
" \"\"\"Constructs a ResNet-18 model.\"\"\"\n",
|
||||||
|
" model = ResNet(block=BasicBlock, \n",
|
||||||
|
" layers=[2, 2, 2, 2],\n",
|
||||||
|
" num_classes=NUM_CLASSES,\n",
|
||||||
|
" grayscale=GRAYSCALE)\n",
|
||||||
|
" return model\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "_lza9t_uj5w1"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"torch.manual_seed(RANDOM_SEED)\n",
|
||||||
|
"\n",
|
||||||
|
"model = resnet18(NUM_CLASSES)\n",
|
||||||
|
"model.to(DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "RAodboScj5w6"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 1547
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 2384585,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976888520,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "Dzh3ROmRj5w7",
|
||||||
|
"outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7",
|
||||||
|
"scrolled": false
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 0000/0469 | Cost: 2.5552\n",
|
||||||
|
"Epoch: 001/010 | Batch 0050/0469 | Cost: 0.1593\n",
|
||||||
|
"Epoch: 001/010 | Batch 0100/0469 | Cost: 0.1098\n",
|
||||||
|
"Epoch: 001/010 | Batch 0150/0469 | Cost: 0.0836\n",
|
||||||
|
"Epoch: 001/010 | Batch 0200/0469 | Cost: 0.0846\n",
|
||||||
|
"Epoch: 001/010 | Batch 0250/0469 | Cost: 0.1061\n",
|
||||||
|
"Epoch: 001/010 | Batch 0300/0469 | Cost: 0.1361\n",
|
||||||
|
"Epoch: 001/010 | Batch 0350/0469 | Cost: 0.0166\n",
|
||||||
|
"Epoch: 001/010 | Batch 0400/0469 | Cost: 0.1310\n",
|
||||||
|
"Epoch: 001/010 | Batch 0450/0469 | Cost: 0.1225\n",
|
||||||
|
"Epoch: 001/010 | Train: 97.822%\n",
|
||||||
|
"Time elapsed: 0.36 min\n",
|
||||||
|
"Epoch: 002/010 | Batch 0000/0469 | Cost: 0.1019\n",
|
||||||
|
"Epoch: 002/010 | Batch 0050/0469 | Cost: 0.0406\n",
|
||||||
|
"Epoch: 002/010 | Batch 0100/0469 | Cost: 0.0383\n",
|
||||||
|
"Epoch: 002/010 | Batch 0150/0469 | Cost: 0.0501\n",
|
||||||
|
"Epoch: 002/010 | Batch 0200/0469 | Cost: 0.0945\n",
|
||||||
|
"Epoch: 002/010 | Batch 0250/0469 | Cost: 0.0163\n",
|
||||||
|
"Epoch: 002/010 | Batch 0300/0469 | Cost: 0.0213\n",
|
||||||
|
"Epoch: 002/010 | Batch 0350/0469 | Cost: 0.0348\n",
|
||||||
|
"Epoch: 002/010 | Batch 0400/0469 | Cost: 0.0236\n",
|
||||||
|
"Epoch: 002/010 | Batch 0450/0469 | Cost: 0.0249\n",
|
||||||
|
"Epoch: 002/010 | Train: 97.227%\n",
|
||||||
|
"Time elapsed: 0.74 min\n",
|
||||||
|
"Epoch: 003/010 | Batch 0000/0469 | Cost: 0.0535\n",
|
||||||
|
"Epoch: 003/010 | Batch 0050/0469 | Cost: 0.0187\n",
|
||||||
|
"Epoch: 003/010 | Batch 0100/0469 | Cost: 0.0272\n",
|
||||||
|
"Epoch: 003/010 | Batch 0150/0469 | Cost: 0.0949\n",
|
||||||
|
"Epoch: 003/010 | Batch 0200/0469 | Cost: 0.0341\n",
|
||||||
|
"Epoch: 003/010 | Batch 0250/0469 | Cost: 0.0314\n",
|
||||||
|
"Epoch: 003/010 | Batch 0300/0469 | Cost: 0.0180\n",
|
||||||
|
"Epoch: 003/010 | Batch 0350/0469 | Cost: 0.0214\n",
|
||||||
|
"Epoch: 003/010 | Batch 0400/0469 | Cost: 0.0969\n",
|
||||||
|
"Epoch: 003/010 | Batch 0450/0469 | Cost: 0.0419\n",
|
||||||
|
"Epoch: 003/010 | Train: 98.687%\n",
|
||||||
|
"Time elapsed: 1.07 min\n",
|
||||||
|
"Epoch: 004/010 | Batch 0000/0469 | Cost: 0.0341\n",
|
||||||
|
"Epoch: 004/010 | Batch 0050/0469 | Cost: 0.0065\n",
|
||||||
|
"Epoch: 004/010 | Batch 0100/0469 | Cost: 0.0252\n",
|
||||||
|
"Epoch: 004/010 | Batch 0150/0469 | Cost: 0.0136\n",
|
||||||
|
"Epoch: 004/010 | Batch 0200/0469 | Cost: 0.0950\n",
|
||||||
|
"Epoch: 004/010 | Batch 0250/0469 | Cost: 0.0405\n",
|
||||||
|
"Epoch: 004/010 | Batch 0300/0469 | Cost: 0.0049\n",
|
||||||
|
"Epoch: 004/010 | Batch 0350/0469 | Cost: 0.0050\n",
|
||||||
|
"Epoch: 004/010 | Batch 0400/0469 | Cost: 0.0074\n",
|
||||||
|
"Epoch: 004/010 | Batch 0450/0469 | Cost: 0.0442\n",
|
||||||
|
"Epoch: 004/010 | Train: 99.352%\n",
|
||||||
|
"Time elapsed: 1.41 min\n",
|
||||||
|
"Epoch: 005/010 | Batch 0000/0469 | Cost: 0.0231\n",
|
||||||
|
"Epoch: 005/010 | Batch 0050/0469 | Cost: 0.0157\n",
|
||||||
|
"Epoch: 005/010 | Batch 0100/0469 | Cost: 0.0369\n",
|
||||||
|
"Epoch: 005/010 | Batch 0150/0469 | Cost: 0.0227\n",
|
||||||
|
"Epoch: 005/010 | Batch 0200/0469 | Cost: 0.0801\n",
|
||||||
|
"Epoch: 005/010 | Batch 0250/0469 | Cost: 0.0293\n",
|
||||||
|
"Epoch: 005/010 | Batch 0300/0469 | Cost: 0.0252\n",
|
||||||
|
"Epoch: 005/010 | Batch 0350/0469 | Cost: 0.0079\n",
|
||||||
|
"Epoch: 005/010 | Batch 0400/0469 | Cost: 0.0813\n",
|
||||||
|
"Epoch: 005/010 | Batch 0450/0469 | Cost: 0.0304\n",
|
||||||
|
"Epoch: 005/010 | Train: 99.385%\n",
|
||||||
|
"Time elapsed: 1.76 min\n",
|
||||||
|
"Epoch: 006/010 | Batch 0000/0469 | Cost: 0.0110\n",
|
||||||
|
"Epoch: 006/010 | Batch 0050/0469 | Cost: 0.0402\n",
|
||||||
|
"Epoch: 006/010 | Batch 0100/0469 | Cost: 0.0013\n",
|
||||||
|
"Epoch: 006/010 | Batch 0150/0469 | Cost: 0.0032\n",
|
||||||
|
"Epoch: 006/010 | Batch 0200/0469 | Cost: 0.0288\n",
|
||||||
|
"Epoch: 006/010 | Batch 0250/0469 | Cost: 0.0733\n",
|
||||||
|
"Epoch: 006/010 | Batch 0300/0469 | Cost: 0.1322\n",
|
||||||
|
"Epoch: 006/010 | Batch 0350/0469 | Cost: 0.0026\n",
|
||||||
|
"Epoch: 006/010 | Batch 0400/0469 | Cost: 0.0504\n",
|
||||||
|
"Epoch: 006/010 | Batch 0450/0469 | Cost: 0.0512\n",
|
||||||
|
"Epoch: 006/010 | Train: 99.250%\n",
|
||||||
|
"Time elapsed: 2.13 min\n",
|
||||||
|
"Epoch: 007/010 | Batch 0000/0469 | Cost: 0.0273\n",
|
||||||
|
"Epoch: 007/010 | Batch 0050/0469 | Cost: 0.0214\n",
|
||||||
|
"Epoch: 007/010 | Batch 0100/0469 | Cost: 0.0034\n",
|
||||||
|
"Epoch: 007/010 | Batch 0150/0469 | Cost: 0.0036\n",
|
||||||
|
"Epoch: 007/010 | Batch 0200/0469 | Cost: 0.0019\n",
|
||||||
|
"Epoch: 007/010 | Batch 0250/0469 | Cost: 0.0437\n",
|
||||||
|
"Epoch: 007/010 | Batch 0300/0469 | Cost: 0.0076\n",
|
||||||
|
"Epoch: 007/010 | Batch 0350/0469 | Cost: 0.0311\n",
|
||||||
|
"Epoch: 007/010 | Batch 0400/0469 | Cost: 0.0146\n",
|
||||||
|
"Epoch: 007/010 | Batch 0450/0469 | Cost: 0.0008\n",
|
||||||
|
"Epoch: 007/010 | Train: 99.147%\n",
|
||||||
|
"Time elapsed: 2.49 min\n",
|
||||||
|
"Epoch: 008/010 | Batch 0000/0469 | Cost: 0.0550\n",
|
||||||
|
"Epoch: 008/010 | Batch 0050/0469 | Cost: 0.0357\n",
|
||||||
|
"Epoch: 008/010 | Batch 0100/0469 | Cost: 0.0021\n",
|
||||||
|
"Epoch: 008/010 | Batch 0150/0469 | Cost: 0.0570\n",
|
||||||
|
"Epoch: 008/010 | Batch 0200/0469 | Cost: 0.0040\n",
|
||||||
|
"Epoch: 008/010 | Batch 0250/0469 | Cost: 0.0118\n",
|
||||||
|
"Epoch: 008/010 | Batch 0300/0469 | Cost: 0.0097\n",
|
||||||
|
"Epoch: 008/010 | Batch 0350/0469 | Cost: 0.0011\n",
|
||||||
|
"Epoch: 008/010 | Batch 0400/0469 | Cost: 0.0399\n",
|
||||||
|
"Epoch: 008/010 | Batch 0450/0469 | Cost: 0.0395\n",
|
||||||
|
"Epoch: 008/010 | Train: 99.360%\n",
|
||||||
|
"Time elapsed: 2.86 min\n",
|
||||||
|
"Epoch: 009/010 | Batch 0000/0469 | Cost: 0.0060\n",
|
||||||
|
"Epoch: 009/010 | Batch 0050/0469 | Cost: 0.0824\n",
|
||||||
|
"Epoch: 009/010 | Batch 0100/0469 | Cost: 0.0235\n",
|
||||||
|
"Epoch: 009/010 | Batch 0150/0469 | Cost: 0.0135\n",
|
||||||
|
"Epoch: 009/010 | Batch 0200/0469 | Cost: 0.0273\n",
|
||||||
|
"Epoch: 009/010 | Batch 0250/0469 | Cost: 0.0391\n",
|
||||||
|
"Epoch: 009/010 | Batch 0300/0469 | Cost: 0.0624\n",
|
||||||
|
"Epoch: 009/010 | Batch 0350/0469 | Cost: 0.0203\n",
|
||||||
|
"Epoch: 009/010 | Batch 0400/0469 | Cost: 0.0012\n",
|
||||||
|
"Epoch: 009/010 | Batch 0450/0469 | Cost: 0.0095\n",
|
||||||
|
"Epoch: 009/010 | Train: 99.480%\n",
|
||||||
|
"Time elapsed: 3.22 min\n",
|
||||||
|
"Epoch: 010/010 | Batch 0000/0469 | Cost: 0.0043\n",
|
||||||
|
"Epoch: 010/010 | Batch 0050/0469 | Cost: 0.0057\n",
|
||||||
|
"Epoch: 010/010 | Batch 0100/0469 | Cost: 0.0165\n",
|
||||||
|
"Epoch: 010/010 | Batch 0150/0469 | Cost: 0.0011\n",
|
||||||
|
"Epoch: 010/010 | Batch 0200/0469 | Cost: 0.0006\n",
|
||||||
|
"Epoch: 010/010 | Batch 0250/0469 | Cost: 0.0017\n",
|
||||||
|
"Epoch: 010/010 | Batch 0300/0469 | Cost: 0.0226\n",
|
||||||
|
"Epoch: 010/010 | Batch 0350/0469 | Cost: 0.0282\n",
|
||||||
|
"Epoch: 010/010 | Batch 0400/0469 | Cost: 0.0430\n",
|
||||||
|
"Epoch: 010/010 | Batch 0450/0469 | Cost: 0.0077\n",
|
||||||
|
"Epoch: 010/010 | Train: 99.522%\n",
|
||||||
|
"Time elapsed: 3.59 min\n",
|
||||||
|
"Total Training Time: 3.59 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader, device):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for i, (features, targets) in enumerate(data_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(NUM_EPOCHS):\n",
|
||||||
|
" \n",
|
||||||
|
" model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" \n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, NUM_EPOCHS, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Epoch: %03d/%03d | Train: %.3f%%' % (\n",
|
||||||
|
" epoch+1, NUM_EPOCHS, \n",
|
||||||
|
" compute_accuracy(model, train_loader, device=DEVICE)))\n",
|
||||||
|
" \n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "paaeEQHQj5xC"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 34
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 6514,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976895054,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "gzQMWKq5j5xE",
|
||||||
|
"outputId": "de7dc005-5eeb-4177-9f9f-d9b5d1358db9"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 99.06%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader, device=DEVICE)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADU5JREFUeJzt3W+IXfWdx/HPZ2OjwRZ1zGwc0ujEIuuouMkyxGDD0qXbYLUQ80DpKCWL0vRBlS32gX/2wUZBDMu2NQ+WwnQTE7Vru9DGRJC12bBiChocZVZNXXc0TklC/kxIMVaEavLdB3PSnercc6/337mT7/sFw9x7vufPl0M+Offe353zc0QIQD5/VnUDAKpB+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJHVONw+2cOHCGBwc7OYhgVQmJyd1/PhxN7JuS+G3fYOkTZLmSfrXiNhYtv7g4KDGxsZaOSSAEsPDww2v2/TLftvzJP2LpK9LukrSiO2rmt0fgO5q5T3/CklvR8T+iPiDpJ9JWtOetgB0WivhXyzpwIznB4tlf8L2ettjtsempqZaOByAdur4p/0RMRoRwxEx3N/f3+nDAWhQK+E/JGnJjOdfLJYBmANaCf/Lkq6wvdT2fEnflLSzPW0B6LSmh/oi4mPbd0l6TtNDfVsiYl/bOgPQUS2N80fEs5KebVMvALqIr/cCSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QVEuz9NqelPS+pFOSPo6I4XY0BaDzWgp/4W8i4ngb9gOgi3jZDyTVavhD0q9sv2J7fTsaAtAdrb7sXxURh2z/uaRdtv8nIl6YuULxn8J6Sbr00ktbPByAdmnpyh8Rh4rfxyRtl7RilnVGI2I4Iob7+/tbORyANmo6/LbPt/2FM48lrZb0RrsaA9BZrbzsXyRpu+0z+/m3iPiPtnQFoOOaDn9E7Jf0l23sBUAXMdQHJEX4gaQIP5AU4QeSIvxAUoQfSKodf9WXwksvvVSztmnTptJtFy9eXFpfsGBBaX3dunWl9b6+vqZqyI0rP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kxTh/g8rG2icmJjp67Icffri0fsEFF9SsrVy5st3tzBmDg4M1a/fff3/pthluOceVH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSYpy/QU8//XTN2vj4eOm2V199dWl93759pfW9e/eW1nfs2FGz9txzz5Vuu3Tp0tL6u+++W1pvxTnnlP/zGxgYKK0fOHCg6WOXfQdAku69996m9z1XcOUHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaTqjvPb3iLpG5KORcQ1xbI+ST+XNChpUtKtEfG7zrVZvaGhoaZqjbj22mtL6yMjI6X1jRs31qxNTk6WbltvnH///v2l9VbMnz+/tF5vnL9e71NTUzVrV155Zem2GTRy5d8q6YZPLLtP0u6IuELS7uI5gDmkbvgj4gVJJz6xeI2kbcXjbZJubnNfADqs2ff8iyLicPH4iKRFbeoHQJe0/IFfRISkqFW3vd72mO2xsvdgALqr2fAftT0gScXvY7VWjIjRiBiOiOH+/v4mDweg3ZoN/05JZ25nu05S7T8rA9CT6obf9lOSXpT0F7YP2r5T0kZJX7M9Ielvi+cA5pC64/wRUWuQ+att7gVNOu+882rWWh3PbvU7DK2odx+D48ePl9avu+66mrXVq1c31dPZhG/4AUkRfiApwg8kRfiBpAg/kBThB5Li1t2ozAcffFBaX7t2bWn99OnTpfVHH320Zm3BggWl22bAlR9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmKcH5XZunVraf3IkSOl9Ysvvri0ftlll33WllLhyg8kRfiBpAg/kBThB5Ii/EBShB9IivADSTHOj4565513atbuueeelvb94osvltYvueSSlvZ/tuPKDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJ1R3nt71F0jckHYuIa4plGyR9W9JUsdoDEfFsp5rE3PXMM8/UrH300Uel295yyy2l9csvv7ypnjCtkSv/Vkk3zLL8RxGxrPgh+MAcUzf8EfGCpBNd6AVAF7Xynv8u26/Z3mL7orZ1BKArmg3/jyV9SdIySYcl/aDWirbX2x6zPTY1NVVrNQBd1lT4I+JoRJyKiNOSfiJpRcm6oxExHBHD/f39zfYJoM2aCr/tgRlP10p6oz3tAOiWRob6npL0FUkLbR+U9I+SvmJ7maSQNCnpOx3sEUAH1A1/RIzMsnhzB3rBHFRvrH779u01a+eee27pto888khpfd68eaV1lOMbfkBShB9IivADSRF+ICnCDyRF+IGkuHU3WrJ5c/mo7549e2rWbrvtttJt+ZPdzuLKDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJMc6PUuPj46X1u+++u7R+4YUX1qw99NBDTfWE9uDKDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJMc6f3IcfflhaHxmZ7c7t/+/UqVOl9dtvv71mjb/XrxZXfiApwg8kRfiBpAg/kBThB5Ii/EBShB9Iqu44v+0lkh6XtEhSSBqNiE22+yT9XNKgpElJt0bE7zrXKppx+vTp0vpNN91UWn/rrbdK60NDQ6X1Bx98sLSO6jRy5f9Y0vcj4ipJKyV91/ZVku6TtDsirpC0u3gOYI6oG/6IOBwRrxaP35f0pqTFktZI2lastk3SzZ1qEkD7fab3/LYHJS2XtFfSoog4XJSOaPptAYA5ouHw2/68pF9I+l5EnJxZi4jQ9OcBs2233vaY7bGpqamWmgXQPg2F3/bnNB38n0bEL4vFR20PFPUBScdm2zYiRiNiOCKG+/v729EzgDaoG37blrRZ0psR8cMZpZ2S1hWP10na0f72AHRKI3/S+2VJ35L0uu0z93F+QNJGSf9u+05Jv5V0a2daRCtOnDhRWn/++edb2v8TTzxRWu/r62tp/+icuuGPiF9Lco3yV9vbDoBu4Rt+QFKEH0iK8ANJEX4gKcIPJEX4gaS4dfdZ4L333qtZW7lyZUv7fvLJJ0vry5cvb2n/qA5XfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IinH+s8Bjjz1Ws7Z///6W9r1q1arS+vS9XjAXceUHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQY558DJiYmSusbNmzoTiM4q3DlB5Ii/EBShB9IivADSRF+ICnCDyRF+IGk6o7z214i6XFJiySFpNGI2GR7g6RvS5oqVn0gIp7tVKOZ7dmzp7R+8uTJpvc9NDRUWl+wYEHT+0Zva+RLPh9L+n5EvGr7C5Jesb2rqP0oIv65c+0B6JS64Y+Iw5IOF4/ft/2mpMWdbgxAZ32m9/y2ByUtl7S3WHSX7ddsb7F9UY1t1tsesz02NTU12yoAKtBw+G1/XtIvJH0vIk5K+rGkL0lapulXBj+YbbuIGI2I4YgY7u/vb0PLANqhofDb/pymg//TiPilJEXE0Yg4FRGnJf1E0orOtQmg3eqG39O3Z90s6c2I+OGM5QMzVlsr6Y32twegUxr5tP/Lkr4l6XXb48WyBySN2F6m6eG/SUnf6UiHaMn1119fWt+1a1dpnaG+s1cjn/b/WtJsN2dnTB+Yw/iGH5AU4QeSIvxAUoQfSIrwA0kRfiApbt09B9xxxx0t1YHZcOUHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQcEd07mD0l6bczFi2UdLxrDXw2vdpbr/Yl0Vuz2tnbZRHR0P3yuhr+Tx3cHouI4coaKNGrvfVqXxK9Nauq3njZDyRF+IGkqg7/aMXHL9OrvfVqXxK9NauS3ip9zw+gOlVf+QFUpJLw277B9lu237Z9XxU91GJ70vbrtsdtj1Xcyxbbx2y/MWNZn+1dtieK37NOk1ZRbxtsHyrO3bjtGyvqbYnt/7L9G9v7bP99sbzSc1fSVyXnresv+23Pk/S/kr4m6aCklyWNRMRvutpIDbYnJQ1HROVjwrb/WtLvJT0eEdcUy/5J0omI2Fj8x3lRRNzbI71tkPT7qmduLiaUGZg5s7SkmyX9nSo8dyV93aoKzlsVV/4Vkt6OiP0R8QdJP5O0poI+el5EvCDpxCcWr5G0rXi8TdP/eLquRm89ISIOR8SrxeP3JZ2ZWbrSc1fSVyWqCP9iSQdmPD+o3pryOyT9yvYrttdX3cwsFhXTpkvSEUmLqmxmFnVnbu6mT8ws3TPnrpkZr9uND/w+bVVE/JWkr0v6bvHytifF9Hu2XhquaWjm5m6ZZWbpP6ry3DU743W7VRH+Q5KWzHj+xWJZT4iIQ8XvY5K2q/dmHz56ZpLU4vexivv5o16auXm2maXVA+eul2a8riL8L0u6wvZS2/MlfVPSzgr6+BTb5xcfxMj2+ZJWq/dmH94paV3xeJ2kHRX28id6ZebmWjNLq+Jz13MzXkdE138k3ajpT/zfkfQPVfRQo6/LJf138bOv6t4kPaXpl4EfafqzkTslXSxpt6QJSf8pqa+HentC0uuSXtN00AYq6m2Vpl/SvyZpvPi5sepzV9JXJeeNb/gBSfGBH5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpP4Pc0oGVHoLWbQAAAAASUVORK5CYII=\n",
|
||||||
|
"text/plain": [
|
||||||
|
"<Figure size 432x288 with 1 Axes>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"needs_background": "light"
|
||||||
|
},
|
||||||
|
"output_type": "display_data"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"for batch_idx, (features, targets) in enumerate(test_loader):\n",
|
||||||
|
"\n",
|
||||||
|
" features = features\n",
|
||||||
|
" targets = targets\n",
|
||||||
|
" break\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
"nhwc_img = np.transpose(features[0], axes=(1, 2, 0))\n",
|
||||||
|
"nhw_img = np.squeeze(nhwc_img.numpy(), axis=2)\n",
|
||||||
|
"plt.imshow(nhw_img, cmap='Greys');"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 11,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Probability 7 99.97%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"model.eval()\n",
|
||||||
|
"logits, probas = model(features.to(device)[0, None])\n",
|
||||||
|
"print('Probability 7 %.2f%%' % (probas[0][7]*100))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 12,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"numpy 1.15.4\n",
|
||||||
|
"pandas 0.23.4\n",
|
||||||
|
"torch 1.0.0\n",
|
||||||
|
"PIL.Image 5.3.0\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"accelerator": "GPU",
|
||||||
|
"colab": {
|
||||||
|
"collapsed_sections": [],
|
||||||
|
"default_view": {},
|
||||||
|
"name": "convnet-vgg16.ipynb",
|
||||||
|
"provenance": [],
|
||||||
|
"version": "0.3.2",
|
||||||
|
"views": {}
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.1"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": true,
|
||||||
|
"toc_position": {
|
||||||
|
"height": "calc(100% - 180px)",
|
||||||
|
"left": "10px",
|
||||||
|
"top": "150px",
|
||||||
|
"width": "371px"
|
||||||
|
},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,912 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "UEBilEjLj5wY"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 119
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 536,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974472601,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "GOzuY8Yvj5wb",
|
||||||
|
"outputId": "c19362ce-f87a-4cc2-84cc-8d7b4b9e6007"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.6.8\n",
|
||||||
|
"IPython 7.2.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.0.0\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "rH4XmErYj5wm"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- ResNet-34 CIFAR-10 Classifier with Pinned Memory"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"This is an example notebook comparing the speed of model training with and without using page-locked memory. Page-locked memory can be enabled by setting `pin_memory=True` in PyTorch's `DataLoader` class (disabled by default).\n",
|
||||||
|
"\n",
|
||||||
|
"Theoretically, pinning the memory should speed up the data transfer rate but minimizing the data transfer cost between CPU and the CUDA device; hence, enabling `pin_memory=True` should make the model training faster by some small margin.\n",
|
||||||
|
"\n",
|
||||||
|
"> Host (CPU) data allocations are pageable by default. The GPU cannot access data directly from pageable host memory, so when a data transfer from pageable host memory to device memory is invoked, the CUDA driver must first allocate a temporary page-locked, or “pinned”, host array, copy the host data to the pinned array, and then transfer the data from the pinned array to device memory, as illustrated below... (Source: https://devblogs.nvidia.com/how-optimize-data-transfers-cuda-cc/)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"After the Model preamble, this Notebook is divided into too subsections, \"Training Without Pinned Memory\" and \"Training with Pinned Memory\" to investigate whether there is a noticable training time difference when toggling `pin_memory` on and off."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Network Architecture"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The network in this notebook is an implementation of the ResNet-34 [1] architecture on the MNIST digits dataset (http://yann.lecun.com/exdb/mnist/) to train a handwritten digit classifier. \n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"References\n",
|
||||||
|
" \n",
|
||||||
|
"- [1] He, K., Zhang, X., Ren, S., & Sun, J. (2016). Deep residual learning for image recognition. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 770-778). ([CVPR Link](https://www.cv-foundation.org/openaccess/content_cvpr_2016/html/He_Deep_Residual_Learning_CVPR_2016_paper.html))\n",
|
||||||
|
"\n",
|
||||||
|
"- [2] http://yann.lecun.com/exdb/mnist/\n",
|
||||||
|
"\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The following figure illustrates residual blocks with skip connections such that the input passed via the shortcut matches the dimensions of the main path's output, which allows the network to learn identity functions.\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"The ResNet-34 architecture actually uses residual blocks with skip connections such that the input passed via the shortcut matches is resized to dimensions of the main path's output. Such a residual block is illustrated below:\n",
|
||||||
|
"\n",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"For a more detailed explanation see the other notebook, [resnet-ex-1.ipynb](resnet-ex-1.ipynb)."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MkoGLH_Tj5wn"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "ORj09gnrj5wp"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import time\n",
|
||||||
|
"\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn as nn\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"from PIL import Image\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "I6hghKPxj5w0"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Model Settings"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 85
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 23936,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974497505,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "NnT0sZIwj5wu",
|
||||||
|
"outputId": "55aed925-d17e-4c6a-8c71-0d9b3bde5637"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"RANDOM_SEED = 1\n",
|
||||||
|
"LEARNING_RATE = 0.001\n",
|
||||||
|
"BATCH_SIZE = 256\n",
|
||||||
|
"NUM_EPOCHS = 10\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"NUM_FEATURES = 28*28\n",
|
||||||
|
"NUM_CLASSES = 10\n",
|
||||||
|
"\n",
|
||||||
|
"# Other\n",
|
||||||
|
"DEVICE = \"cuda:1\"\n",
|
||||||
|
"GRAYSCALE = False"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The following code cell that implements the ResNet-34 architecture is a derivative of the code provided at https://pytorch.org/docs/0.4.0/_modules/torchvision/models/resnet.html."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def conv3x3(in_planes, out_planes, stride=1):\n",
|
||||||
|
" \"\"\"3x3 convolution with padding\"\"\"\n",
|
||||||
|
" return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
|
||||||
|
" padding=1, bias=False)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class BasicBlock(nn.Module):\n",
|
||||||
|
" expansion = 1\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, inplanes, planes, stride=1, downsample=None):\n",
|
||||||
|
" super(BasicBlock, self).__init__()\n",
|
||||||
|
" self.conv1 = conv3x3(inplanes, planes, stride)\n",
|
||||||
|
" self.bn1 = nn.BatchNorm2d(planes)\n",
|
||||||
|
" self.relu = nn.ReLU(inplace=True)\n",
|
||||||
|
" self.conv2 = conv3x3(planes, planes)\n",
|
||||||
|
" self.bn2 = nn.BatchNorm2d(planes)\n",
|
||||||
|
" self.downsample = downsample\n",
|
||||||
|
" self.stride = stride\n",
|
||||||
|
"\n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" residual = x\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv1(x)\n",
|
||||||
|
" out = self.bn1(out)\n",
|
||||||
|
" out = self.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv2(out)\n",
|
||||||
|
" out = self.bn2(out)\n",
|
||||||
|
"\n",
|
||||||
|
" if self.downsample is not None:\n",
|
||||||
|
" residual = self.downsample(x)\n",
|
||||||
|
"\n",
|
||||||
|
" out += residual\n",
|
||||||
|
" out = self.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" return out\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class ResNet(nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, block, layers, num_classes, grayscale):\n",
|
||||||
|
" self.inplanes = 64\n",
|
||||||
|
" if grayscale:\n",
|
||||||
|
" in_dim = 1\n",
|
||||||
|
" else:\n",
|
||||||
|
" in_dim = 3\n",
|
||||||
|
" super(ResNet, self).__init__()\n",
|
||||||
|
" self.conv1 = nn.Conv2d(in_dim, 64, kernel_size=7, stride=2, padding=3,\n",
|
||||||
|
" bias=False)\n",
|
||||||
|
" self.bn1 = nn.BatchNorm2d(64)\n",
|
||||||
|
" self.relu = nn.ReLU(inplace=True)\n",
|
||||||
|
" self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
|
||||||
|
" self.layer1 = self._make_layer(block, 64, layers[0])\n",
|
||||||
|
" self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n",
|
||||||
|
" self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n",
|
||||||
|
" self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n",
|
||||||
|
" self.avgpool = nn.AvgPool2d(7, stride=1)\n",
|
||||||
|
" self.fc = nn.Linear(512 * block.expansion, num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
" for m in self.modules():\n",
|
||||||
|
" if isinstance(m, nn.Conv2d):\n",
|
||||||
|
" n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
|
||||||
|
" m.weight.data.normal_(0, (2. / n)**.5)\n",
|
||||||
|
" elif isinstance(m, nn.BatchNorm2d):\n",
|
||||||
|
" m.weight.data.fill_(1)\n",
|
||||||
|
" m.bias.data.zero_()\n",
|
||||||
|
"\n",
|
||||||
|
" def _make_layer(self, block, planes, blocks, stride=1):\n",
|
||||||
|
" downsample = None\n",
|
||||||
|
" if stride != 1 or self.inplanes != planes * block.expansion:\n",
|
||||||
|
" downsample = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(self.inplanes, planes * block.expansion,\n",
|
||||||
|
" kernel_size=1, stride=stride, bias=False),\n",
|
||||||
|
" nn.BatchNorm2d(planes * block.expansion),\n",
|
||||||
|
" )\n",
|
||||||
|
"\n",
|
||||||
|
" layers = []\n",
|
||||||
|
" layers.append(block(self.inplanes, planes, stride, downsample))\n",
|
||||||
|
" self.inplanes = planes * block.expansion\n",
|
||||||
|
" for i in range(1, blocks):\n",
|
||||||
|
" layers.append(block(self.inplanes, planes))\n",
|
||||||
|
"\n",
|
||||||
|
" return nn.Sequential(*layers)\n",
|
||||||
|
"\n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" x = self.conv1(x)\n",
|
||||||
|
" x = self.bn1(x)\n",
|
||||||
|
" x = self.relu(x)\n",
|
||||||
|
" x = self.maxpool(x)\n",
|
||||||
|
"\n",
|
||||||
|
" x = self.layer1(x)\n",
|
||||||
|
" x = self.layer2(x)\n",
|
||||||
|
" x = self.layer3(x)\n",
|
||||||
|
" x = self.layer4(x)\n",
|
||||||
|
" # because MNIST is already 1x1 here:\n",
|
||||||
|
" # disable avg pooling\n",
|
||||||
|
" #x = self.avgpool(x)\n",
|
||||||
|
" \n",
|
||||||
|
" x = x.view(x.size(0), -1)\n",
|
||||||
|
" logits = self.fc(x)\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def resnet34(num_classes):\n",
|
||||||
|
" \"\"\"Constructs a ResNet-34 model.\"\"\"\n",
|
||||||
|
" model = ResNet(block=BasicBlock, \n",
|
||||||
|
" layers=[3, 4, 6, 3],\n",
|
||||||
|
" num_classes=NUM_CLASSES,\n",
|
||||||
|
" grayscale=GRAYSCALE)\n",
|
||||||
|
" return model\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "RAodboScj5w6"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Training without Pinned Memory"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Files already downloaded and verified\n",
|
||||||
|
"Image batch dimensions: torch.Size([256, 3, 32, 32])\n",
|
||||||
|
"Image label dimensions: torch.Size([256])\n",
|
||||||
|
"Image batch dimensions: torch.Size([256, 3, 32, 32])\n",
|
||||||
|
"Image label dimensions: torch.Size([256])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### CIFAR-10 Dataset\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE, \n",
|
||||||
|
" num_workers=8,\n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE,\n",
|
||||||
|
" num_workers=8,\n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "_lza9t_uj5w1"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"torch.manual_seed(RANDOM_SEED)\n",
|
||||||
|
"\n",
|
||||||
|
"model = resnet34(NUM_CLASSES)\n",
|
||||||
|
"model.to(DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 1547
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 2384585,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976888520,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "Dzh3ROmRj5w7",
|
||||||
|
"outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7",
|
||||||
|
"scrolled": false
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 0000/0196 | Cost: 2.6021\n",
|
||||||
|
"Epoch: 001/010 | Batch 0150/0196 | Cost: 1.3961\n",
|
||||||
|
"Epoch: 001/010 | Train: 45.084%\n",
|
||||||
|
"Time elapsed: 0.26 min\n",
|
||||||
|
"Epoch: 002/010 | Batch 0000/0196 | Cost: 1.1228\n",
|
||||||
|
"Epoch: 002/010 | Batch 0150/0196 | Cost: 1.0426\n",
|
||||||
|
"Epoch: 002/010 | Train: 56.166%\n",
|
||||||
|
"Time elapsed: 0.52 min\n",
|
||||||
|
"Epoch: 003/010 | Batch 0000/0196 | Cost: 0.9980\n",
|
||||||
|
"Epoch: 003/010 | Batch 0150/0196 | Cost: 0.8279\n",
|
||||||
|
"Epoch: 003/010 | Train: 66.702%\n",
|
||||||
|
"Time elapsed: 0.80 min\n",
|
||||||
|
"Epoch: 004/010 | Batch 0000/0196 | Cost: 0.6384\n",
|
||||||
|
"Epoch: 004/010 | Batch 0150/0196 | Cost: 0.7103\n",
|
||||||
|
"Epoch: 004/010 | Train: 65.330%\n",
|
||||||
|
"Time elapsed: 1.08 min\n",
|
||||||
|
"Epoch: 005/010 | Batch 0000/0196 | Cost: 0.6308\n",
|
||||||
|
"Epoch: 005/010 | Batch 0150/0196 | Cost: 0.5913\n",
|
||||||
|
"Epoch: 005/010 | Train: 79.636%\n",
|
||||||
|
"Time elapsed: 1.36 min\n",
|
||||||
|
"Epoch: 006/010 | Batch 0000/0196 | Cost: 0.4409\n",
|
||||||
|
"Epoch: 006/010 | Batch 0150/0196 | Cost: 0.5557\n",
|
||||||
|
"Epoch: 006/010 | Train: 76.456%\n",
|
||||||
|
"Time elapsed: 1.62 min\n",
|
||||||
|
"Epoch: 007/010 | Batch 0000/0196 | Cost: 0.4778\n",
|
||||||
|
"Epoch: 007/010 | Batch 0150/0196 | Cost: 0.4815\n",
|
||||||
|
"Epoch: 007/010 | Train: 65.890%\n",
|
||||||
|
"Time elapsed: 1.89 min\n",
|
||||||
|
"Epoch: 008/010 | Batch 0000/0196 | Cost: 0.3782\n",
|
||||||
|
"Epoch: 008/010 | Batch 0150/0196 | Cost: 0.4339\n",
|
||||||
|
"Epoch: 008/010 | Train: 85.200%\n",
|
||||||
|
"Time elapsed: 2.16 min\n",
|
||||||
|
"Epoch: 009/010 | Batch 0000/0196 | Cost: 0.3083\n",
|
||||||
|
"Epoch: 009/010 | Batch 0150/0196 | Cost: 0.3290\n",
|
||||||
|
"Epoch: 009/010 | Train: 78.108%\n",
|
||||||
|
"Time elapsed: 2.42 min\n",
|
||||||
|
"Epoch: 010/010 | Batch 0000/0196 | Cost: 0.2229\n",
|
||||||
|
"Epoch: 010/010 | Batch 0150/0196 | Cost: 0.1945\n",
|
||||||
|
"Epoch: 010/010 | Train: 87.384%\n",
|
||||||
|
"Time elapsed: 2.70 min\n",
|
||||||
|
"Total Training Time: 2.70 min\n",
|
||||||
|
"Test accuracy: 70.67%\n",
|
||||||
|
"Total Time: 2.71 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader, device):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for i, (features, targets) in enumerate(data_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(NUM_EPOCHS):\n",
|
||||||
|
" \n",
|
||||||
|
" model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" \n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 150:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, NUM_EPOCHS, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Epoch: %03d/%03d | Train: %.3f%%' % (\n",
|
||||||
|
" epoch+1, NUM_EPOCHS, \n",
|
||||||
|
" compute_accuracy(model, train_loader, device=DEVICE)))\n",
|
||||||
|
" \n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader, device=DEVICE)))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Training with Pinned Memory"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Files already downloaded and verified\n",
|
||||||
|
"Image batch dimensions: torch.Size([256, 3, 32, 32])\n",
|
||||||
|
"Image label dimensions: torch.Size([256])\n",
|
||||||
|
"Image batch dimensions: torch.Size([256, 3, 32, 32])\n",
|
||||||
|
"Image label dimensions: torch.Size([256])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### CIFAR-10 Dataset\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE, \n",
|
||||||
|
" pin_memory=True,\n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE,\n",
|
||||||
|
" pin_memory=True,\n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "_lza9t_uj5w1"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"torch.manual_seed(RANDOM_SEED)\n",
|
||||||
|
"\n",
|
||||||
|
"model = resnet34(NUM_CLASSES)\n",
|
||||||
|
"model.to(DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 1547
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 2384585,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976888520,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "Dzh3ROmRj5w7",
|
||||||
|
"outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7",
|
||||||
|
"scrolled": false
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 0000/0196 | Cost: 2.6021\n",
|
||||||
|
"Epoch: 001/010 | Batch 0150/0196 | Cost: 1.3961\n",
|
||||||
|
"Epoch: 001/010 | Train: 45.084%\n",
|
||||||
|
"Time elapsed: 0.39 min\n",
|
||||||
|
"Epoch: 002/010 | Batch 0000/0196 | Cost: 1.1228\n",
|
||||||
|
"Epoch: 002/010 | Batch 0150/0196 | Cost: 1.0426\n",
|
||||||
|
"Epoch: 002/010 | Train: 56.166%\n",
|
||||||
|
"Time elapsed: 0.77 min\n",
|
||||||
|
"Epoch: 003/010 | Batch 0000/0196 | Cost: 0.9980\n",
|
||||||
|
"Epoch: 003/010 | Batch 0150/0196 | Cost: 0.8279\n",
|
||||||
|
"Epoch: 003/010 | Train: 66.702%\n",
|
||||||
|
"Time elapsed: 1.16 min\n",
|
||||||
|
"Epoch: 004/010 | Batch 0000/0196 | Cost: 0.6384\n",
|
||||||
|
"Epoch: 004/010 | Batch 0150/0196 | Cost: 0.7103\n",
|
||||||
|
"Epoch: 004/010 | Train: 65.330%\n",
|
||||||
|
"Time elapsed: 1.55 min\n",
|
||||||
|
"Epoch: 005/010 | Batch 0000/0196 | Cost: 0.6308\n",
|
||||||
|
"Epoch: 005/010 | Batch 0150/0196 | Cost: 0.5913\n",
|
||||||
|
"Epoch: 005/010 | Train: 79.636%\n",
|
||||||
|
"Time elapsed: 1.94 min\n",
|
||||||
|
"Epoch: 006/010 | Batch 0000/0196 | Cost: 0.4409\n",
|
||||||
|
"Epoch: 006/010 | Batch 0150/0196 | Cost: 0.5557\n",
|
||||||
|
"Epoch: 006/010 | Train: 76.456%\n",
|
||||||
|
"Time elapsed: 2.33 min\n",
|
||||||
|
"Epoch: 007/010 | Batch 0000/0196 | Cost: 0.4778\n",
|
||||||
|
"Epoch: 007/010 | Batch 0150/0196 | Cost: 0.4815\n",
|
||||||
|
"Epoch: 007/010 | Train: 65.890%\n",
|
||||||
|
"Time elapsed: 2.71 min\n",
|
||||||
|
"Epoch: 008/010 | Batch 0000/0196 | Cost: 0.3782\n",
|
||||||
|
"Epoch: 008/010 | Batch 0150/0196 | Cost: 0.4339\n",
|
||||||
|
"Epoch: 008/010 | Train: 85.200%\n",
|
||||||
|
"Time elapsed: 3.10 min\n",
|
||||||
|
"Epoch: 009/010 | Batch 0000/0196 | Cost: 0.3083\n",
|
||||||
|
"Epoch: 009/010 | Batch 0150/0196 | Cost: 0.3290\n",
|
||||||
|
"Epoch: 009/010 | Train: 78.108%\n",
|
||||||
|
"Time elapsed: 3.49 min\n",
|
||||||
|
"Epoch: 010/010 | Batch 0000/0196 | Cost: 0.2229\n",
|
||||||
|
"Epoch: 010/010 | Batch 0150/0196 | Cost: 0.1945\n",
|
||||||
|
"Epoch: 010/010 | Train: 87.384%\n",
|
||||||
|
"Time elapsed: 3.88 min\n",
|
||||||
|
"Total Training Time: 3.88 min\n",
|
||||||
|
"Test accuracy: 70.67%\n",
|
||||||
|
"Total Time: 3.91 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader, device):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for i, (features, targets) in enumerate(data_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(NUM_EPOCHS):\n",
|
||||||
|
" \n",
|
||||||
|
" model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" \n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 150:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, NUM_EPOCHS, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Epoch: %03d/%03d | Train: %.3f%%' % (\n",
|
||||||
|
" epoch+1, NUM_EPOCHS, \n",
|
||||||
|
" compute_accuracy(model, train_loader, device=DEVICE)))\n",
|
||||||
|
" \n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader, device=DEVICE)))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Conclusions"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Based on the training time without and with `pin_memory=True`, there doesn't seem to be a speed-up when using page-locked (or \"pinned\") memory -- in fact, pinning the memory even slowed down the training. (I reran the code in the opposite order, i.e., `pin_memory=True` first, and got the same results.) This could be due to the relatively small dataset size, batch size, and hardware configuration that I was using:\n",
|
||||||
|
"\n",
|
||||||
|
"- Processor: Intel Xeon® Processor E5-2650 v4 (12 core)\n",
|
||||||
|
"- GPU: NVIDIA GeForce GTX 1080Ti\n",
|
||||||
|
"- Motherboard: ASUS X99-E-10G WS with PCI-E Gen3 X16 port\n",
|
||||||
|
"- Memory: 128 GB DDR4 RAM\n",
|
||||||
|
"- Storage: SSD"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 11,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"numpy 1.15.4\n",
|
||||||
|
"pandas 0.23.4\n",
|
||||||
|
"torch 1.0.0\n",
|
||||||
|
"PIL.Image 5.3.0\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"accelerator": "GPU",
|
||||||
|
"colab": {
|
||||||
|
"collapsed_sections": [],
|
||||||
|
"default_view": {},
|
||||||
|
"name": "convnet-vgg16.ipynb",
|
||||||
|
"provenance": [],
|
||||||
|
"version": "0.3.2",
|
||||||
|
"views": {}
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.1"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": true,
|
||||||
|
"toc_position": {
|
||||||
|
"height": "calc(100% - 180px)",
|
||||||
|
"left": "10px",
|
||||||
|
"top": "150px",
|
||||||
|
"width": "371px"
|
||||||
|
},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
@@ -0,0 +1,850 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "UEBilEjLj5wY"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 119
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 536,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974472601,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "GOzuY8Yvj5wb",
|
||||||
|
"outputId": "c19362ce-f87a-4cc2-84cc-8d7b4b9e6007"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.6.8\n",
|
||||||
|
"IPython 7.2.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.0.0\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "rH4XmErYj5wm"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- ResNet-34 MNIST Digits Classifier"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Network Architecture"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The network in this notebook is an implementation of the ResNet-34 [1] architecture on the MNIST digits dataset (http://yann.lecun.com/exdb/mnist/) to train a handwritten digit classifier. \n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"References\n",
|
||||||
|
" \n",
|
||||||
|
"- [1] He, K., Zhang, X., Ren, S., & Sun, J. (2016). Deep residual learning for image recognition. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 770-778). ([CVPR Link](https://www.cv-foundation.org/openaccess/content_cvpr_2016/html/He_Deep_Residual_Learning_CVPR_2016_paper.html))\n",
|
||||||
|
"\n",
|
||||||
|
"- [2] http://yann.lecun.com/exdb/mnist/\n",
|
||||||
|
"\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The following figure illustrates residual blocks with skip connections such that the input passed via the shortcut matches the dimensions of the main path's output, which allows the network to learn identity functions.\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"The ResNet-34 architecture actually uses residual blocks with skip connections such that the input passed via the shortcut matches is resized to dimensions of the main path's output. Such a residual block is illustrated below:\n",
|
||||||
|
"\n",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"For a more detailed explanation see the other notebook, [resnet-ex-1.ipynb](resnet-ex-1.ipynb)."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MkoGLH_Tj5wn"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "ORj09gnrj5wp"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import time\n",
|
||||||
|
"\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn as nn\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"from PIL import Image\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "I6hghKPxj5w0"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Model Settings"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 85
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 23936,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974497505,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "NnT0sZIwj5wu",
|
||||||
|
"outputId": "55aed925-d17e-4c6a-8c71-0d9b3bde5637"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"RANDOM_SEED = 1\n",
|
||||||
|
"LEARNING_RATE = 0.001\n",
|
||||||
|
"BATCH_SIZE = 128\n",
|
||||||
|
"NUM_EPOCHS = 10\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"NUM_FEATURES = 28*28\n",
|
||||||
|
"NUM_CLASSES = 10\n",
|
||||||
|
"\n",
|
||||||
|
"# Other\n",
|
||||||
|
"DEVICE = \"cuda:2\"\n",
|
||||||
|
"GRAYSCALE = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### MNIST Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 1 | Batch index: 0 | Batch size: 128\n",
|
||||||
|
"Epoch: 2 | Batch index: 0 | Batch size: 128\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"device = torch.device(DEVICE)\n",
|
||||||
|
"torch.manual_seed(0)\n",
|
||||||
|
"\n",
|
||||||
|
"for epoch in range(2):\n",
|
||||||
|
"\n",
|
||||||
|
" for batch_idx, (x, y) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" print('Epoch:', epoch+1, end='')\n",
|
||||||
|
" print(' | Batch index:', batch_idx, end='')\n",
|
||||||
|
" print(' | Batch size:', y.size()[0])\n",
|
||||||
|
" \n",
|
||||||
|
" x = x.to(device)\n",
|
||||||
|
" y = y.to(device)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The following code cell that implements the ResNet-34 architecture is a derivative of the code provided at https://pytorch.org/docs/0.4.0/_modules/torchvision/models/resnet.html."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def conv3x3(in_planes, out_planes, stride=1):\n",
|
||||||
|
" \"\"\"3x3 convolution with padding\"\"\"\n",
|
||||||
|
" return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
|
||||||
|
" padding=1, bias=False)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class BasicBlock(nn.Module):\n",
|
||||||
|
" expansion = 1\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, inplanes, planes, stride=1, downsample=None):\n",
|
||||||
|
" super(BasicBlock, self).__init__()\n",
|
||||||
|
" self.conv1 = conv3x3(inplanes, planes, stride)\n",
|
||||||
|
" self.bn1 = nn.BatchNorm2d(planes)\n",
|
||||||
|
" self.relu = nn.ReLU(inplace=True)\n",
|
||||||
|
" self.conv2 = conv3x3(planes, planes)\n",
|
||||||
|
" self.bn2 = nn.BatchNorm2d(planes)\n",
|
||||||
|
" self.downsample = downsample\n",
|
||||||
|
" self.stride = stride\n",
|
||||||
|
"\n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" residual = x\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv1(x)\n",
|
||||||
|
" out = self.bn1(out)\n",
|
||||||
|
" out = self.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv2(out)\n",
|
||||||
|
" out = self.bn2(out)\n",
|
||||||
|
"\n",
|
||||||
|
" if self.downsample is not None:\n",
|
||||||
|
" residual = self.downsample(x)\n",
|
||||||
|
"\n",
|
||||||
|
" out += residual\n",
|
||||||
|
" out = self.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" return out\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class ResNet(nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, block, layers, num_classes, grayscale):\n",
|
||||||
|
" self.inplanes = 64\n",
|
||||||
|
" if grayscale:\n",
|
||||||
|
" in_dim = 1\n",
|
||||||
|
" else:\n",
|
||||||
|
" in_dim = 3\n",
|
||||||
|
" super(ResNet, self).__init__()\n",
|
||||||
|
" self.conv1 = nn.Conv2d(in_dim, 64, kernel_size=7, stride=2, padding=3,\n",
|
||||||
|
" bias=False)\n",
|
||||||
|
" self.bn1 = nn.BatchNorm2d(64)\n",
|
||||||
|
" self.relu = nn.ReLU(inplace=True)\n",
|
||||||
|
" self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
|
||||||
|
" self.layer1 = self._make_layer(block, 64, layers[0])\n",
|
||||||
|
" self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n",
|
||||||
|
" self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n",
|
||||||
|
" self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n",
|
||||||
|
" self.avgpool = nn.AvgPool2d(7, stride=1)\n",
|
||||||
|
" self.fc = nn.Linear(512 * block.expansion, num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
" for m in self.modules():\n",
|
||||||
|
" if isinstance(m, nn.Conv2d):\n",
|
||||||
|
" n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
|
||||||
|
" m.weight.data.normal_(0, (2. / n)**.5)\n",
|
||||||
|
" elif isinstance(m, nn.BatchNorm2d):\n",
|
||||||
|
" m.weight.data.fill_(1)\n",
|
||||||
|
" m.bias.data.zero_()\n",
|
||||||
|
"\n",
|
||||||
|
" def _make_layer(self, block, planes, blocks, stride=1):\n",
|
||||||
|
" downsample = None\n",
|
||||||
|
" if stride != 1 or self.inplanes != planes * block.expansion:\n",
|
||||||
|
" downsample = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(self.inplanes, planes * block.expansion,\n",
|
||||||
|
" kernel_size=1, stride=stride, bias=False),\n",
|
||||||
|
" nn.BatchNorm2d(planes * block.expansion),\n",
|
||||||
|
" )\n",
|
||||||
|
"\n",
|
||||||
|
" layers = []\n",
|
||||||
|
" layers.append(block(self.inplanes, planes, stride, downsample))\n",
|
||||||
|
" self.inplanes = planes * block.expansion\n",
|
||||||
|
" for i in range(1, blocks):\n",
|
||||||
|
" layers.append(block(self.inplanes, planes))\n",
|
||||||
|
"\n",
|
||||||
|
" return nn.Sequential(*layers)\n",
|
||||||
|
"\n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" x = self.conv1(x)\n",
|
||||||
|
" x = self.bn1(x)\n",
|
||||||
|
" x = self.relu(x)\n",
|
||||||
|
" x = self.maxpool(x)\n",
|
||||||
|
"\n",
|
||||||
|
" x = self.layer1(x)\n",
|
||||||
|
" x = self.layer2(x)\n",
|
||||||
|
" x = self.layer3(x)\n",
|
||||||
|
" x = self.layer4(x)\n",
|
||||||
|
" # because MNIST is already 1x1 here:\n",
|
||||||
|
" # disable avg pooling\n",
|
||||||
|
" #x = self.avgpool(x)\n",
|
||||||
|
" \n",
|
||||||
|
" x = x.view(x.size(0), -1)\n",
|
||||||
|
" logits = self.fc(x)\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def resnet34(num_classes):\n",
|
||||||
|
" \"\"\"Constructs a ResNet-34 model.\"\"\"\n",
|
||||||
|
" model = ResNet(block=BasicBlock, \n",
|
||||||
|
" layers=[3, 4, 6, 3],\n",
|
||||||
|
" num_classes=NUM_CLASSES,\n",
|
||||||
|
" grayscale=GRAYSCALE)\n",
|
||||||
|
" return model\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "_lza9t_uj5w1"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"torch.manual_seed(RANDOM_SEED)\n",
|
||||||
|
"model = resnet34(NUM_CLASSES)\n",
|
||||||
|
"model.to(DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "RAodboScj5w6"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 1547
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 2384585,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976888520,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "Dzh3ROmRj5w7",
|
||||||
|
"outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7",
|
||||||
|
"scrolled": false
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 0000/0469 | Cost: 2.8909\n",
|
||||||
|
"Epoch: 001/010 | Batch 0050/0469 | Cost: 0.2777\n",
|
||||||
|
"Epoch: 001/010 | Batch 0100/0469 | Cost: 0.0824\n",
|
||||||
|
"Epoch: 001/010 | Batch 0150/0469 | Cost: 0.1159\n",
|
||||||
|
"Epoch: 001/010 | Batch 0200/0469 | Cost: 0.1098\n",
|
||||||
|
"Epoch: 001/010 | Batch 0250/0469 | Cost: 0.2297\n",
|
||||||
|
"Epoch: 001/010 | Batch 0300/0469 | Cost: 0.0692\n",
|
||||||
|
"Epoch: 001/010 | Batch 0350/0469 | Cost: 0.0762\n",
|
||||||
|
"Epoch: 001/010 | Batch 0400/0469 | Cost: 0.0318\n",
|
||||||
|
"Epoch: 001/010 | Batch 0450/0469 | Cost: 0.0387\n",
|
||||||
|
"Epoch: 001/010 | Train: 96.232%\n",
|
||||||
|
"Time elapsed: 1.35 min\n",
|
||||||
|
"Epoch: 002/010 | Batch 0000/0469 | Cost: 0.1717\n",
|
||||||
|
"Epoch: 002/010 | Batch 0050/0469 | Cost: 0.0508\n",
|
||||||
|
"Epoch: 002/010 | Batch 0100/0469 | Cost: 0.1568\n",
|
||||||
|
"Epoch: 002/010 | Batch 0150/0469 | Cost: 0.0505\n",
|
||||||
|
"Epoch: 002/010 | Batch 0200/0469 | Cost: 0.0380\n",
|
||||||
|
"Epoch: 002/010 | Batch 0250/0469 | Cost: 0.0550\n",
|
||||||
|
"Epoch: 002/010 | Batch 0300/0469 | Cost: 0.0708\n",
|
||||||
|
"Epoch: 002/010 | Batch 0350/0469 | Cost: 0.0737\n",
|
||||||
|
"Epoch: 002/010 | Batch 0400/0469 | Cost: 0.0399\n",
|
||||||
|
"Epoch: 002/010 | Batch 0450/0469 | Cost: 0.0172\n",
|
||||||
|
"Epoch: 002/010 | Train: 96.182%\n",
|
||||||
|
"Time elapsed: 2.69 min\n",
|
||||||
|
"Epoch: 003/010 | Batch 0000/0469 | Cost: 0.1397\n",
|
||||||
|
"Epoch: 003/010 | Batch 0050/0469 | Cost: 0.0358\n",
|
||||||
|
"Epoch: 003/010 | Batch 0100/0469 | Cost: 0.0052\n",
|
||||||
|
"Epoch: 003/010 | Batch 0150/0469 | Cost: 0.0488\n",
|
||||||
|
"Epoch: 003/010 | Batch 0200/0469 | Cost: 0.0314\n",
|
||||||
|
"Epoch: 003/010 | Batch 0250/0469 | Cost: 0.0550\n",
|
||||||
|
"Epoch: 003/010 | Batch 0300/0469 | Cost: 0.0239\n",
|
||||||
|
"Epoch: 003/010 | Batch 0350/0469 | Cost: 0.0566\n",
|
||||||
|
"Epoch: 003/010 | Batch 0400/0469 | Cost: 0.0222\n",
|
||||||
|
"Epoch: 003/010 | Batch 0450/0469 | Cost: 0.0515\n",
|
||||||
|
"Epoch: 003/010 | Train: 98.672%\n",
|
||||||
|
"Time elapsed: 3.97 min\n",
|
||||||
|
"Epoch: 004/010 | Batch 0000/0469 | Cost: 0.0364\n",
|
||||||
|
"Epoch: 004/010 | Batch 0050/0469 | Cost: 0.0026\n",
|
||||||
|
"Epoch: 004/010 | Batch 0100/0469 | Cost: 0.0321\n",
|
||||||
|
"Epoch: 004/010 | Batch 0150/0469 | Cost: 0.0180\n",
|
||||||
|
"Epoch: 004/010 | Batch 0200/0469 | Cost: 0.0851\n",
|
||||||
|
"Epoch: 004/010 | Batch 0250/0469 | Cost: 0.0142\n",
|
||||||
|
"Epoch: 004/010 | Batch 0300/0469 | Cost: 0.0362\n",
|
||||||
|
"Epoch: 004/010 | Batch 0350/0469 | Cost: 0.0563\n",
|
||||||
|
"Epoch: 004/010 | Batch 0400/0469 | Cost: 0.0512\n",
|
||||||
|
"Epoch: 004/010 | Batch 0450/0469 | Cost: 0.0353\n",
|
||||||
|
"Epoch: 004/010 | Train: 98.750%\n",
|
||||||
|
"Time elapsed: 5.22 min\n",
|
||||||
|
"Epoch: 005/010 | Batch 0000/0469 | Cost: 0.0242\n",
|
||||||
|
"Epoch: 005/010 | Batch 0050/0469 | Cost: 0.0092\n",
|
||||||
|
"Epoch: 005/010 | Batch 0100/0469 | Cost: 0.0055\n",
|
||||||
|
"Epoch: 005/010 | Batch 0150/0469 | Cost: 0.0129\n",
|
||||||
|
"Epoch: 005/010 | Batch 0200/0469 | Cost: 0.0259\n",
|
||||||
|
"Epoch: 005/010 | Batch 0250/0469 | Cost: 0.0256\n",
|
||||||
|
"Epoch: 005/010 | Batch 0300/0469 | Cost: 0.0082\n",
|
||||||
|
"Epoch: 005/010 | Batch 0350/0469 | Cost: 0.0493\n",
|
||||||
|
"Epoch: 005/010 | Batch 0400/0469 | Cost: 0.0026\n",
|
||||||
|
"Epoch: 005/010 | Batch 0450/0469 | Cost: 0.0212\n",
|
||||||
|
"Epoch: 005/010 | Train: 98.642%\n",
|
||||||
|
"Time elapsed: 6.48 min\n",
|
||||||
|
"Epoch: 006/010 | Batch 0000/0469 | Cost: 0.0437\n",
|
||||||
|
"Epoch: 006/010 | Batch 0050/0469 | Cost: 0.0071\n",
|
||||||
|
"Epoch: 006/010 | Batch 0100/0469 | Cost: 0.0274\n",
|
||||||
|
"Epoch: 006/010 | Batch 0150/0469 | Cost: 0.0300\n",
|
||||||
|
"Epoch: 006/010 | Batch 0200/0469 | Cost: 0.0169\n",
|
||||||
|
"Epoch: 006/010 | Batch 0250/0469 | Cost: 0.0176\n",
|
||||||
|
"Epoch: 006/010 | Batch 0300/0469 | Cost: 0.0036\n",
|
||||||
|
"Epoch: 006/010 | Batch 0350/0469 | Cost: 0.0473\n",
|
||||||
|
"Epoch: 006/010 | Batch 0400/0469 | Cost: 0.0090\n",
|
||||||
|
"Epoch: 006/010 | Batch 0450/0469 | Cost: 0.0848\n",
|
||||||
|
"Epoch: 006/010 | Train: 97.143%\n",
|
||||||
|
"Time elapsed: 7.73 min\n",
|
||||||
|
"Epoch: 007/010 | Batch 0000/0469 | Cost: 0.0441\n",
|
||||||
|
"Epoch: 007/010 | Batch 0050/0469 | Cost: 0.0150\n",
|
||||||
|
"Epoch: 007/010 | Batch 0100/0469 | Cost: 0.0407\n",
|
||||||
|
"Epoch: 007/010 | Batch 0150/0469 | Cost: 0.0082\n",
|
||||||
|
"Epoch: 007/010 | Batch 0200/0469 | Cost: 0.0643\n",
|
||||||
|
"Epoch: 007/010 | Batch 0250/0469 | Cost: 0.0132\n",
|
||||||
|
"Epoch: 007/010 | Batch 0300/0469 | Cost: 0.0054\n",
|
||||||
|
"Epoch: 007/010 | Batch 0350/0469 | Cost: 0.0046\n",
|
||||||
|
"Epoch: 007/010 | Batch 0400/0469 | Cost: 0.0143\n",
|
||||||
|
"Epoch: 007/010 | Batch 0450/0469 | Cost: 0.0397\n",
|
||||||
|
"Epoch: 007/010 | Train: 99.555%\n",
|
||||||
|
"Time elapsed: 8.98 min\n",
|
||||||
|
"Epoch: 008/010 | Batch 0000/0469 | Cost: 0.0115\n",
|
||||||
|
"Epoch: 008/010 | Batch 0050/0469 | Cost: 0.0036\n",
|
||||||
|
"Epoch: 008/010 | Batch 0100/0469 | Cost: 0.0046\n",
|
||||||
|
"Epoch: 008/010 | Batch 0150/0469 | Cost: 0.0028\n",
|
||||||
|
"Epoch: 008/010 | Batch 0200/0469 | Cost: 0.0080\n",
|
||||||
|
"Epoch: 008/010 | Batch 0250/0469 | Cost: 0.0143\n",
|
||||||
|
"Epoch: 008/010 | Batch 0300/0469 | Cost: 0.0091\n",
|
||||||
|
"Epoch: 008/010 | Batch 0350/0469 | Cost: 0.0122\n",
|
||||||
|
"Epoch: 008/010 | Batch 0400/0469 | Cost: 0.0372\n",
|
||||||
|
"Epoch: 008/010 | Batch 0450/0469 | Cost: 0.0093\n",
|
||||||
|
"Epoch: 008/010 | Train: 99.615%\n",
|
||||||
|
"Time elapsed: 10.24 min\n",
|
||||||
|
"Epoch: 009/010 | Batch 0000/0469 | Cost: 0.0032\n",
|
||||||
|
"Epoch: 009/010 | Batch 0050/0469 | Cost: 0.0009\n",
|
||||||
|
"Epoch: 009/010 | Batch 0100/0469 | Cost: 0.0091\n",
|
||||||
|
"Epoch: 009/010 | Batch 0150/0469 | Cost: 0.0601\n",
|
||||||
|
"Epoch: 009/010 | Batch 0200/0469 | Cost: 0.0274\n",
|
||||||
|
"Epoch: 009/010 | Batch 0250/0469 | Cost: 0.0127\n",
|
||||||
|
"Epoch: 009/010 | Batch 0300/0469 | Cost: 0.0147\n",
|
||||||
|
"Epoch: 009/010 | Batch 0350/0469 | Cost: 0.0501\n",
|
||||||
|
"Epoch: 009/010 | Batch 0400/0469 | Cost: 0.0198\n",
|
||||||
|
"Epoch: 009/010 | Batch 0450/0469 | Cost: 0.0020\n",
|
||||||
|
"Epoch: 009/010 | Train: 99.357%\n",
|
||||||
|
"Time elapsed: 11.50 min\n",
|
||||||
|
"Epoch: 010/010 | Batch 0000/0469 | Cost: 0.0073\n",
|
||||||
|
"Epoch: 010/010 | Batch 0050/0469 | Cost: 0.0077\n",
|
||||||
|
"Epoch: 010/010 | Batch 0100/0469 | Cost: 0.0079\n",
|
||||||
|
"Epoch: 010/010 | Batch 0150/0469 | Cost: 0.0546\n",
|
||||||
|
"Epoch: 010/010 | Batch 0200/0469 | Cost: 0.0021\n",
|
||||||
|
"Epoch: 010/010 | Batch 0250/0469 | Cost: 0.0211\n",
|
||||||
|
"Epoch: 010/010 | Batch 0300/0469 | Cost: 0.0018\n",
|
||||||
|
"Epoch: 010/010 | Batch 0350/0469 | Cost: 0.0042\n",
|
||||||
|
"Epoch: 010/010 | Batch 0400/0469 | Cost: 0.0078\n",
|
||||||
|
"Epoch: 010/010 | Batch 0450/0469 | Cost: 0.0017\n",
|
||||||
|
"Epoch: 010/010 | Train: 99.532%\n",
|
||||||
|
"Time elapsed: 12.75 min\n",
|
||||||
|
"Total Training Time: 12.75 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader, device):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for i, (features, targets) in enumerate(data_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(NUM_EPOCHS):\n",
|
||||||
|
" \n",
|
||||||
|
" model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" \n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, NUM_EPOCHS, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Epoch: %03d/%03d | Train: %.3f%%' % (\n",
|
||||||
|
" epoch+1, NUM_EPOCHS, \n",
|
||||||
|
" compute_accuracy(model, train_loader, device=DEVICE)))\n",
|
||||||
|
" \n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "paaeEQHQj5xC"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 34
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 6514,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976895054,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "gzQMWKq5j5xE",
|
||||||
|
"outputId": "de7dc005-5eeb-4177-9f9f-d9b5d1358db9"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 99.04%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader, device=DEVICE)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADU5JREFUeJzt3W+IXfWdx/HPZ2OjwRZ1zGwc0ujEIuuouMkyxGDD0qXbYLUQ80DpKCWL0vRBlS32gX/2wUZBDMu2NQ+WwnQTE7Vru9DGRJC12bBiChocZVZNXXc0TklC/kxIMVaEavLdB3PSnercc6/337mT7/sFw9x7vufPl0M+Offe353zc0QIQD5/VnUDAKpB+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJHVONw+2cOHCGBwc7OYhgVQmJyd1/PhxN7JuS+G3fYOkTZLmSfrXiNhYtv7g4KDGxsZaOSSAEsPDww2v2/TLftvzJP2LpK9LukrSiO2rmt0fgO5q5T3/CklvR8T+iPiDpJ9JWtOetgB0WivhXyzpwIznB4tlf8L2ettjtsempqZaOByAdur4p/0RMRoRwxEx3N/f3+nDAWhQK+E/JGnJjOdfLJYBmANaCf/Lkq6wvdT2fEnflLSzPW0B6LSmh/oi4mPbd0l6TtNDfVsiYl/bOgPQUS2N80fEs5KebVMvALqIr/cCSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QVEuz9NqelPS+pFOSPo6I4XY0BaDzWgp/4W8i4ngb9gOgi3jZDyTVavhD0q9sv2J7fTsaAtAdrb7sXxURh2z/uaRdtv8nIl6YuULxn8J6Sbr00ktbPByAdmnpyh8Rh4rfxyRtl7RilnVGI2I4Iob7+/tbORyANmo6/LbPt/2FM48lrZb0RrsaA9BZrbzsXyRpu+0z+/m3iPiPtnQFoOOaDn9E7Jf0l23sBUAXMdQHJEX4gaQIP5AU4QeSIvxAUoQfSKodf9WXwksvvVSztmnTptJtFy9eXFpfsGBBaX3dunWl9b6+vqZqyI0rP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kxTh/g8rG2icmJjp67Icffri0fsEFF9SsrVy5st3tzBmDg4M1a/fff3/pthluOceVH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSYpy/QU8//XTN2vj4eOm2V199dWl93759pfW9e/eW1nfs2FGz9txzz5Vuu3Tp0tL6u+++W1pvxTnnlP/zGxgYKK0fOHCg6WOXfQdAku69996m9z1XcOUHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaTqjvPb3iLpG5KORcQ1xbI+ST+XNChpUtKtEfG7zrVZvaGhoaZqjbj22mtL6yMjI6X1jRs31qxNTk6WbltvnH///v2l9VbMnz+/tF5vnL9e71NTUzVrV155Zem2GTRy5d8q6YZPLLtP0u6IuELS7uI5gDmkbvgj4gVJJz6xeI2kbcXjbZJubnNfADqs2ff8iyLicPH4iKRFbeoHQJe0/IFfRISkqFW3vd72mO2xsvdgALqr2fAftT0gScXvY7VWjIjRiBiOiOH+/v4mDweg3ZoN/05JZ25nu05S7T8rA9CT6obf9lOSXpT0F7YP2r5T0kZJX7M9Ielvi+cA5pC64/wRUWuQ+att7gVNOu+882rWWh3PbvU7DK2odx+D48ePl9avu+66mrXVq1c31dPZhG/4AUkRfiApwg8kRfiBpAg/kBThB5Li1t2ozAcffFBaX7t2bWn99OnTpfVHH320Zm3BggWl22bAlR9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmKcH5XZunVraf3IkSOl9Ysvvri0ftlll33WllLhyg8kRfiBpAg/kBThB5Ii/EBShB9IivADSTHOj4565513atbuueeelvb94osvltYvueSSlvZ/tuPKDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJ1R3nt71F0jckHYuIa4plGyR9W9JUsdoDEfFsp5rE3PXMM8/UrH300Uel295yyy2l9csvv7ypnjCtkSv/Vkk3zLL8RxGxrPgh+MAcUzf8EfGCpBNd6AVAF7Xynv8u26/Z3mL7orZ1BKArmg3/jyV9SdIySYcl/aDWirbX2x6zPTY1NVVrNQBd1lT4I+JoRJyKiNOSfiJpRcm6oxExHBHD/f39zfYJoM2aCr/tgRlP10p6oz3tAOiWRob6npL0FUkLbR+U9I+SvmJ7maSQNCnpOx3sEUAH1A1/RIzMsnhzB3rBHFRvrH779u01a+eee27pto888khpfd68eaV1lOMbfkBShB9IivADSRF+ICnCDyRF+IGkuHU3WrJ5c/mo7549e2rWbrvtttJt+ZPdzuLKDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJMc6PUuPj46X1u+++u7R+4YUX1qw99NBDTfWE9uDKDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJMc6f3IcfflhaHxmZ7c7t/+/UqVOl9dtvv71mjb/XrxZXfiApwg8kRfiBpAg/kBThB5Ii/EBShB9Iqu44v+0lkh6XtEhSSBqNiE22+yT9XNKgpElJt0bE7zrXKppx+vTp0vpNN91UWn/rrbdK60NDQ6X1Bx98sLSO6jRy5f9Y0vcj4ipJKyV91/ZVku6TtDsirpC0u3gOYI6oG/6IOBwRrxaP35f0pqTFktZI2lastk3SzZ1qEkD7fab3/LYHJS2XtFfSoog4XJSOaPptAYA5ouHw2/68pF9I+l5EnJxZi4jQ9OcBs2233vaY7bGpqamWmgXQPg2F3/bnNB38n0bEL4vFR20PFPUBScdm2zYiRiNiOCKG+/v729EzgDaoG37blrRZ0psR8cMZpZ2S1hWP10na0f72AHRKI3/S+2VJ35L0uu0z93F+QNJGSf9u+05Jv5V0a2daRCtOnDhRWn/++edb2v8TTzxRWu/r62tp/+icuuGPiF9Lco3yV9vbDoBu4Rt+QFKEH0iK8ANJEX4gKcIPJEX4gaS4dfdZ4L333qtZW7lyZUv7fvLJJ0vry5cvb2n/qA5XfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IinH+s8Bjjz1Ws7Z///6W9r1q1arS+vS9XjAXceUHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQY558DJiYmSusbNmzoTiM4q3DlB5Ii/EBShB9IivADSRF+ICnCDyRF+IGk6o7z214i6XFJiySFpNGI2GR7g6RvS5oqVn0gIp7tVKOZ7dmzp7R+8uTJpvc9NDRUWl+wYEHT+0Zva+RLPh9L+n5EvGr7C5Jesb2rqP0oIv65c+0B6JS64Y+Iw5IOF4/ft/2mpMWdbgxAZ32m9/y2ByUtl7S3WHSX7ddsb7F9UY1t1tsesz02NTU12yoAKtBw+G1/XtIvJH0vIk5K+rGkL0lapulXBj+YbbuIGI2I4YgY7u/vb0PLANqhofDb/pymg//TiPilJEXE0Yg4FRGnJf1E0orOtQmg3eqG39O3Z90s6c2I+OGM5QMzVlsr6Y32twegUxr5tP/Lkr4l6XXb48WyBySN2F6m6eG/SUnf6UiHaMn1119fWt+1a1dpnaG+s1cjn/b/WtJsN2dnTB+Yw/iGH5AU4QeSIvxAUoQfSIrwA0kRfiApbt09B9xxxx0t1YHZcOUHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQcEd07mD0l6bczFi2UdLxrDXw2vdpbr/Yl0Vuz2tnbZRHR0P3yuhr+Tx3cHouI4coaKNGrvfVqXxK9Nauq3njZDyRF+IGkqg7/aMXHL9OrvfVqXxK9NauS3ip9zw+gOlVf+QFUpJLw277B9lu237Z9XxU91GJ70vbrtsdtj1Xcyxbbx2y/MWNZn+1dtieK37NOk1ZRbxtsHyrO3bjtGyvqbYnt/7L9G9v7bP99sbzSc1fSVyXnresv+23Pk/S/kr4m6aCklyWNRMRvutpIDbYnJQ1HROVjwrb/WtLvJT0eEdcUy/5J0omI2Fj8x3lRRNzbI71tkPT7qmduLiaUGZg5s7SkmyX9nSo8dyV93aoKzlsVV/4Vkt6OiP0R8QdJP5O0poI+el5EvCDpxCcWr5G0rXi8TdP/eLquRm89ISIOR8SrxeP3JZ2ZWbrSc1fSVyWqCP9iSQdmPD+o3pryOyT9yvYrttdX3cwsFhXTpkvSEUmLqmxmFnVnbu6mT8ws3TPnrpkZr9uND/w+bVVE/JWkr0v6bvHytifF9Hu2XhquaWjm5m6ZZWbpP6ry3DU743W7VRH+Q5KWzHj+xWJZT4iIQ8XvY5K2q/dmHz56ZpLU4vexivv5o16auXm2maXVA+eul2a8riL8L0u6wvZS2/MlfVPSzgr6+BTb5xcfxMj2+ZJWq/dmH94paV3xeJ2kHRX28id6ZebmWjNLq+Jz13MzXkdE138k3ajpT/zfkfQPVfRQo6/LJf138bOv6t4kPaXpl4EfafqzkTslXSxpt6QJSf8pqa+HentC0uuSXtN00AYq6m2Vpl/SvyZpvPi5sepzV9JXJeeNb/gBSfGBH5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpP4Pc0oGVHoLWbQAAAAASUVORK5CYII=\n",
|
||||||
|
"text/plain": [
|
||||||
|
"<Figure size 432x288 with 1 Axes>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"needs_background": "light"
|
||||||
|
},
|
||||||
|
"output_type": "display_data"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"for batch_idx, (features, targets) in enumerate(test_loader):\n",
|
||||||
|
"\n",
|
||||||
|
" features = features\n",
|
||||||
|
" targets = targets\n",
|
||||||
|
" break\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
"nhwc_img = np.transpose(features[0], axes=(1, 2, 0))\n",
|
||||||
|
"nhw_img = np.squeeze(nhwc_img.numpy(), axis=2)\n",
|
||||||
|
"plt.imshow(nhw_img, cmap='Greys');"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 11,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Probability 7 100.00%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"model.eval()\n",
|
||||||
|
"logits, probas = model(features.to(device)[0, None])\n",
|
||||||
|
"print('Probability 7 %.2f%%' % (probas[0][7]*100))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 12,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"numpy 1.15.4\n",
|
||||||
|
"pandas 0.23.4\n",
|
||||||
|
"torch 1.0.0\n",
|
||||||
|
"PIL.Image 5.3.0\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"accelerator": "GPU",
|
||||||
|
"colab": {
|
||||||
|
"collapsed_sections": [],
|
||||||
|
"default_view": {},
|
||||||
|
"name": "convnet-vgg16.ipynb",
|
||||||
|
"provenance": [],
|
||||||
|
"version": "0.3.2",
|
||||||
|
"views": {}
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.1"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": true,
|
||||||
|
"toc_position": {
|
||||||
|
"height": "calc(100% - 180px)",
|
||||||
|
"left": "10px",
|
||||||
|
"top": "150px",
|
||||||
|
"width": "371px"
|
||||||
|
},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,990 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "UEBilEjLj5wY"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 119
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 536,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974472601,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "GOzuY8Yvj5wb",
|
||||||
|
"outputId": "c19362ce-f87a-4cc2-84cc-8d7b4b9e6007"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.6.8\n",
|
||||||
|
"IPython 7.2.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.0.0\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "rH4XmErYj5wm"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- CNN Gender Classifier (ResNet-50 Architecture, CelebA) with Data Parallelism"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Network Architecture"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The network in this notebook is an implementation of the ResNet-50 [1] architecture on the CelebA face dataset [2] to train a gender classifier. \n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"References\n",
|
||||||
|
" \n",
|
||||||
|
"- [1] He, K., Zhang, X., Ren, S., & Sun, J. (2016). Deep residual learning for image recognition. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 770-778). ([CVPR Link](https://www.cv-foundation.org/openaccess/content_cvpr_2016/html/He_Deep_Residual_Learning_CVPR_2016_paper.html))\n",
|
||||||
|
"\n",
|
||||||
|
"- [2] Zhang, K., Tan, L., Li, Z., & Qiao, Y. (2016). Gender and smile classification using deep convolutional neural networks. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition Workshops (pp. 34-38).\n",
|
||||||
|
"\n",
|
||||||
|
"The ResNet-50 architecture is similar to the ResNet-34 architecture shown below (from [1]):\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"However, in ResNet-50, the skip connection uses a bottleneck (from [1]):\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The following figure illustrates residual blocks with skip connections such that the input passed via the shortcut matches the dimensions of the main path's output, which allows the network to learn identity functions.\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"The ResNet-34 architecture actually uses residual blocks with skip connections such that the input passed via the shortcut matches is resized to dimensions of the main path's output. Such a residual block is illustrated below:\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"The ResNet-50 uses a bottleneck as shown below:\n",
|
||||||
|
"\n",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"For a more detailed explanation see the other notebook, [resnet-ex-1.ipynb](resnet-ex-1.ipynb)."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MkoGLH_Tj5wn"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "ORj09gnrj5wp"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import time\n",
|
||||||
|
"\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn as nn\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"from PIL import Image\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "I6hghKPxj5w0"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Model Settings"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 85
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 23936,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974497505,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "NnT0sZIwj5wu",
|
||||||
|
"outputId": "55aed925-d17e-4c6a-8c71-0d9b3bde5637"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"RANDOM_SEED = 1\n",
|
||||||
|
"LEARNING_RATE = 0.0001\n",
|
||||||
|
"BATCH_SIZE = 128\n",
|
||||||
|
"NUM_EPOCHS = 20\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"NUM_FEATURES = 28*28\n",
|
||||||
|
"NUM_CLASSES = 10\n",
|
||||||
|
"\n",
|
||||||
|
"# Other\n",
|
||||||
|
"DEVICE = \"cuda:0\"\n",
|
||||||
|
"GRAYSCALE = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### MNIST Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=BATCH_SIZE, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 1 | Batch index: 0 | Batch size: 128\n",
|
||||||
|
"Epoch: 2 | Batch index: 0 | Batch size: 128\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"device = torch.device(DEVICE)\n",
|
||||||
|
"torch.manual_seed(0)\n",
|
||||||
|
"\n",
|
||||||
|
"for epoch in range(2):\n",
|
||||||
|
"\n",
|
||||||
|
" for batch_idx, (x, y) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" print('Epoch:', epoch+1, end='')\n",
|
||||||
|
" print(' | Batch index:', batch_idx, end='')\n",
|
||||||
|
" print(' | Batch size:', y.size()[0])\n",
|
||||||
|
" \n",
|
||||||
|
" x = x.to(device)\n",
|
||||||
|
" y = y.to(device)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The following code cell that implements the ResNet-34 architecture is a derivative of the code provided at https://pytorch.org/docs/0.4.0/_modules/torchvision/models/resnet.html."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def conv3x3(in_planes, out_planes, stride=1):\n",
|
||||||
|
" \"\"\"3x3 convolution with padding\"\"\"\n",
|
||||||
|
" return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
|
||||||
|
" padding=1, bias=False)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class Bottleneck(nn.Module):\n",
|
||||||
|
" expansion = 4\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, inplanes, planes, stride=1, downsample=None):\n",
|
||||||
|
" super(Bottleneck, self).__init__()\n",
|
||||||
|
" self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n",
|
||||||
|
" self.bn1 = nn.BatchNorm2d(planes)\n",
|
||||||
|
" self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,\n",
|
||||||
|
" padding=1, bias=False)\n",
|
||||||
|
" self.bn2 = nn.BatchNorm2d(planes)\n",
|
||||||
|
" self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n",
|
||||||
|
" self.bn3 = nn.BatchNorm2d(planes * 4)\n",
|
||||||
|
" self.relu = nn.ReLU(inplace=True)\n",
|
||||||
|
" self.downsample = downsample\n",
|
||||||
|
" self.stride = stride\n",
|
||||||
|
"\n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" residual = x\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv1(x)\n",
|
||||||
|
" out = self.bn1(out)\n",
|
||||||
|
" out = self.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv2(out)\n",
|
||||||
|
" out = self.bn2(out)\n",
|
||||||
|
" out = self.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv3(out)\n",
|
||||||
|
" out = self.bn3(out)\n",
|
||||||
|
"\n",
|
||||||
|
" if self.downsample is not None:\n",
|
||||||
|
" residual = self.downsample(x)\n",
|
||||||
|
"\n",
|
||||||
|
" out += residual\n",
|
||||||
|
" out = self.relu(out)\n",
|
||||||
|
"\n",
|
||||||
|
" return out\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class ResNet(nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, block, layers, num_classes, grayscale):\n",
|
||||||
|
" self.inplanes = 64\n",
|
||||||
|
" if grayscale:\n",
|
||||||
|
" in_dim = 1\n",
|
||||||
|
" else:\n",
|
||||||
|
" in_dim = 3\n",
|
||||||
|
" super(ResNet, self).__init__()\n",
|
||||||
|
" self.conv1 = nn.Conv2d(in_dim, 64, kernel_size=7, stride=2, padding=3,\n",
|
||||||
|
" bias=False)\n",
|
||||||
|
" self.bn1 = nn.BatchNorm2d(64)\n",
|
||||||
|
" self.relu = nn.ReLU(inplace=True)\n",
|
||||||
|
" self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
|
||||||
|
" self.layer1 = self._make_layer(block, 64, layers[0])\n",
|
||||||
|
" self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n",
|
||||||
|
" self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n",
|
||||||
|
" self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n",
|
||||||
|
" self.avgpool = nn.AvgPool2d(7, stride=1)\n",
|
||||||
|
" self.fc = nn.Linear(512 * block.expansion, num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
" for m in self.modules():\n",
|
||||||
|
" if isinstance(m, nn.Conv2d):\n",
|
||||||
|
" n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
|
||||||
|
" m.weight.data.normal_(0, (2. / n)**.5)\n",
|
||||||
|
" elif isinstance(m, nn.BatchNorm2d):\n",
|
||||||
|
" m.weight.data.fill_(1)\n",
|
||||||
|
" m.bias.data.zero_()\n",
|
||||||
|
"\n",
|
||||||
|
" def _make_layer(self, block, planes, blocks, stride=1):\n",
|
||||||
|
" downsample = None\n",
|
||||||
|
" if stride != 1 or self.inplanes != planes * block.expansion:\n",
|
||||||
|
" downsample = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(self.inplanes, planes * block.expansion,\n",
|
||||||
|
" kernel_size=1, stride=stride, bias=False),\n",
|
||||||
|
" nn.BatchNorm2d(planes * block.expansion),\n",
|
||||||
|
" )\n",
|
||||||
|
"\n",
|
||||||
|
" layers = []\n",
|
||||||
|
" layers.append(block(self.inplanes, planes, stride, downsample))\n",
|
||||||
|
" self.inplanes = planes * block.expansion\n",
|
||||||
|
" for i in range(1, blocks):\n",
|
||||||
|
" layers.append(block(self.inplanes, planes))\n",
|
||||||
|
"\n",
|
||||||
|
" return nn.Sequential(*layers)\n",
|
||||||
|
"\n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" x = self.conv1(x)\n",
|
||||||
|
" x = self.bn1(x)\n",
|
||||||
|
" x = self.relu(x)\n",
|
||||||
|
" x = self.maxpool(x)\n",
|
||||||
|
"\n",
|
||||||
|
" x = self.layer1(x)\n",
|
||||||
|
" x = self.layer2(x)\n",
|
||||||
|
" x = self.layer3(x)\n",
|
||||||
|
" x = self.layer4(x)\n",
|
||||||
|
" # because MNIST is already 1x1 here:\n",
|
||||||
|
" # disable avg pooling\n",
|
||||||
|
" #x = self.avgpool(x)\n",
|
||||||
|
" \n",
|
||||||
|
" x = x.view(x.size(0), -1)\n",
|
||||||
|
" logits = self.fc(x)\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def resnet34(num_classes):\n",
|
||||||
|
" \"\"\"Constructs a ResNet-34 model.\"\"\"\n",
|
||||||
|
" model = ResNet(block=Bottleneck, \n",
|
||||||
|
" layers=[3, 4, 6, 3],\n",
|
||||||
|
" num_classes=NUM_CLASSES,\n",
|
||||||
|
" grayscale=GRAYSCALE)\n",
|
||||||
|
" return model\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "_lza9t_uj5w1"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"torch.manual_seed(RANDOM_SEED)\n",
|
||||||
|
"\n",
|
||||||
|
"model = resnet34(NUM_CLASSES)\n",
|
||||||
|
"model.to(DEVICE)\n",
|
||||||
|
" \n",
|
||||||
|
"optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "RAodboScj5w6"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 1547
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 2384585,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976888520,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "Dzh3ROmRj5w7",
|
||||||
|
"outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7",
|
||||||
|
"scrolled": false
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/020 | Batch 0000/0469 | Cost: 2.4814\n",
|
||||||
|
"Epoch: 001/020 | Batch 0050/0469 | Cost: 1.6407\n",
|
||||||
|
"Epoch: 001/020 | Batch 0100/0469 | Cost: 1.2549\n",
|
||||||
|
"Epoch: 001/020 | Batch 0150/0469 | Cost: 0.8013\n",
|
||||||
|
"Epoch: 001/020 | Batch 0200/0469 | Cost: 0.7644\n",
|
||||||
|
"Epoch: 001/020 | Batch 0250/0469 | Cost: 0.6729\n",
|
||||||
|
"Epoch: 001/020 | Batch 0300/0469 | Cost: 0.5566\n",
|
||||||
|
"Epoch: 001/020 | Batch 0350/0469 | Cost: 0.4682\n",
|
||||||
|
"Epoch: 001/020 | Batch 0400/0469 | Cost: 0.3663\n",
|
||||||
|
"Epoch: 001/020 | Batch 0450/0469 | Cost: 0.3904\n",
|
||||||
|
"Epoch: 001/020 | Train: 90.932%\n",
|
||||||
|
"Time elapsed: 0.66 min\n",
|
||||||
|
"Epoch: 002/020 | Batch 0000/0469 | Cost: 0.3419\n",
|
||||||
|
"Epoch: 002/020 | Batch 0050/0469 | Cost: 0.2901\n",
|
||||||
|
"Epoch: 002/020 | Batch 0100/0469 | Cost: 0.1937\n",
|
||||||
|
"Epoch: 002/020 | Batch 0150/0469 | Cost: 0.2761\n",
|
||||||
|
"Epoch: 002/020 | Batch 0200/0469 | Cost: 0.2688\n",
|
||||||
|
"Epoch: 002/020 | Batch 0250/0469 | Cost: 0.1875\n",
|
||||||
|
"Epoch: 002/020 | Batch 0300/0469 | Cost: 0.2401\n",
|
||||||
|
"Epoch: 002/020 | Batch 0350/0469 | Cost: 0.1768\n",
|
||||||
|
"Epoch: 002/020 | Batch 0400/0469 | Cost: 0.2161\n",
|
||||||
|
"Epoch: 002/020 | Batch 0450/0469 | Cost: 0.1624\n",
|
||||||
|
"Epoch: 002/020 | Train: 96.738%\n",
|
||||||
|
"Time elapsed: 1.34 min\n",
|
||||||
|
"Epoch: 003/020 | Batch 0000/0469 | Cost: 0.1413\n",
|
||||||
|
"Epoch: 003/020 | Batch 0050/0469 | Cost: 0.0832\n",
|
||||||
|
"Epoch: 003/020 | Batch 0100/0469 | Cost: 0.0924\n",
|
||||||
|
"Epoch: 003/020 | Batch 0150/0469 | Cost: 0.0587\n",
|
||||||
|
"Epoch: 003/020 | Batch 0200/0469 | Cost: 0.0991\n",
|
||||||
|
"Epoch: 003/020 | Batch 0250/0469 | Cost: 0.1508\n",
|
||||||
|
"Epoch: 003/020 | Batch 0300/0469 | Cost: 0.1367\n",
|
||||||
|
"Epoch: 003/020 | Batch 0350/0469 | Cost: 0.1431\n",
|
||||||
|
"Epoch: 003/020 | Batch 0400/0469 | Cost: 0.1650\n",
|
||||||
|
"Epoch: 003/020 | Batch 0450/0469 | Cost: 0.1842\n",
|
||||||
|
"Epoch: 003/020 | Train: 98.288%\n",
|
||||||
|
"Time elapsed: 2.03 min\n",
|
||||||
|
"Epoch: 004/020 | Batch 0000/0469 | Cost: 0.0812\n",
|
||||||
|
"Epoch: 004/020 | Batch 0050/0469 | Cost: 0.0499\n",
|
||||||
|
"Epoch: 004/020 | Batch 0100/0469 | Cost: 0.0413\n",
|
||||||
|
"Epoch: 004/020 | Batch 0150/0469 | Cost: 0.0929\n",
|
||||||
|
"Epoch: 004/020 | Batch 0200/0469 | Cost: 0.0501\n",
|
||||||
|
"Epoch: 004/020 | Batch 0250/0469 | Cost: 0.1147\n",
|
||||||
|
"Epoch: 004/020 | Batch 0300/0469 | Cost: 0.0277\n",
|
||||||
|
"Epoch: 004/020 | Batch 0350/0469 | Cost: 0.0659\n",
|
||||||
|
"Epoch: 004/020 | Batch 0400/0469 | Cost: 0.0854\n",
|
||||||
|
"Epoch: 004/020 | Batch 0450/0469 | Cost: 0.0368\n",
|
||||||
|
"Epoch: 004/020 | Train: 98.942%\n",
|
||||||
|
"Time elapsed: 2.71 min\n",
|
||||||
|
"Epoch: 005/020 | Batch 0000/0469 | Cost: 0.0120\n",
|
||||||
|
"Epoch: 005/020 | Batch 0050/0469 | Cost: 0.0127\n",
|
||||||
|
"Epoch: 005/020 | Batch 0100/0469 | Cost: 0.0516\n",
|
||||||
|
"Epoch: 005/020 | Batch 0150/0469 | Cost: 0.0341\n",
|
||||||
|
"Epoch: 005/020 | Batch 0200/0469 | Cost: 0.0600\n",
|
||||||
|
"Epoch: 005/020 | Batch 0250/0469 | Cost: 0.1150\n",
|
||||||
|
"Epoch: 005/020 | Batch 0300/0469 | Cost: 0.0312\n",
|
||||||
|
"Epoch: 005/020 | Batch 0350/0469 | Cost: 0.0494\n",
|
||||||
|
"Epoch: 005/020 | Batch 0400/0469 | Cost: 0.0711\n",
|
||||||
|
"Epoch: 005/020 | Batch 0450/0469 | Cost: 0.0531\n",
|
||||||
|
"Epoch: 005/020 | Train: 99.060%\n",
|
||||||
|
"Time elapsed: 3.39 min\n",
|
||||||
|
"Epoch: 006/020 | Batch 0000/0469 | Cost: 0.0589\n",
|
||||||
|
"Epoch: 006/020 | Batch 0050/0469 | Cost: 0.0341\n",
|
||||||
|
"Epoch: 006/020 | Batch 0100/0469 | Cost: 0.0205\n",
|
||||||
|
"Epoch: 006/020 | Batch 0150/0469 | Cost: 0.0219\n",
|
||||||
|
"Epoch: 006/020 | Batch 0200/0469 | Cost: 0.0495\n",
|
||||||
|
"Epoch: 006/020 | Batch 0250/0469 | Cost: 0.0344\n",
|
||||||
|
"Epoch: 006/020 | Batch 0300/0469 | Cost: 0.0298\n",
|
||||||
|
"Epoch: 006/020 | Batch 0350/0469 | Cost: 0.0375\n",
|
||||||
|
"Epoch: 006/020 | Batch 0400/0469 | Cost: 0.1366\n",
|
||||||
|
"Epoch: 006/020 | Batch 0450/0469 | Cost: 0.0469\n",
|
||||||
|
"Epoch: 006/020 | Train: 99.312%\n",
|
||||||
|
"Time elapsed: 4.07 min\n",
|
||||||
|
"Epoch: 007/020 | Batch 0000/0469 | Cost: 0.0116\n",
|
||||||
|
"Epoch: 007/020 | Batch 0050/0469 | Cost: 0.0411\n",
|
||||||
|
"Epoch: 007/020 | Batch 0100/0469 | Cost: 0.0115\n",
|
||||||
|
"Epoch: 007/020 | Batch 0150/0469 | Cost: 0.0110\n",
|
||||||
|
"Epoch: 007/020 | Batch 0200/0469 | Cost: 0.1041\n",
|
||||||
|
"Epoch: 007/020 | Batch 0250/0469 | Cost: 0.0172\n",
|
||||||
|
"Epoch: 007/020 | Batch 0300/0469 | Cost: 0.0614\n",
|
||||||
|
"Epoch: 007/020 | Batch 0350/0469 | Cost: 0.0363\n",
|
||||||
|
"Epoch: 007/020 | Batch 0400/0469 | Cost: 0.0366\n",
|
||||||
|
"Epoch: 007/020 | Batch 0450/0469 | Cost: 0.0660\n",
|
||||||
|
"Epoch: 007/020 | Train: 99.482%\n",
|
||||||
|
"Time elapsed: 4.76 min\n",
|
||||||
|
"Epoch: 008/020 | Batch 0000/0469 | Cost: 0.0132\n",
|
||||||
|
"Epoch: 008/020 | Batch 0050/0469 | Cost: 0.0016\n",
|
||||||
|
"Epoch: 008/020 | Batch 0100/0469 | Cost: 0.0226\n",
|
||||||
|
"Epoch: 008/020 | Batch 0150/0469 | Cost: 0.0283\n",
|
||||||
|
"Epoch: 008/020 | Batch 0200/0469 | Cost: 0.0373\n",
|
||||||
|
"Epoch: 008/020 | Batch 0250/0469 | Cost: 0.0584\n",
|
||||||
|
"Epoch: 008/020 | Batch 0300/0469 | Cost: 0.0115\n",
|
||||||
|
"Epoch: 008/020 | Batch 0350/0469 | Cost: 0.0893\n",
|
||||||
|
"Epoch: 008/020 | Batch 0400/0469 | Cost: 0.0368\n",
|
||||||
|
"Epoch: 008/020 | Batch 0450/0469 | Cost: 0.0184\n",
|
||||||
|
"Epoch: 008/020 | Train: 99.192%\n",
|
||||||
|
"Time elapsed: 5.44 min\n",
|
||||||
|
"Epoch: 009/020 | Batch 0000/0469 | Cost: 0.0047\n",
|
||||||
|
"Epoch: 009/020 | Batch 0050/0469 | Cost: 0.0088\n",
|
||||||
|
"Epoch: 009/020 | Batch 0100/0469 | Cost: 0.0021\n",
|
||||||
|
"Epoch: 009/020 | Batch 0150/0469 | Cost: 0.0861\n",
|
||||||
|
"Epoch: 009/020 | Batch 0200/0469 | Cost: 0.0031\n",
|
||||||
|
"Epoch: 009/020 | Batch 0250/0469 | Cost: 0.0761\n",
|
||||||
|
"Epoch: 009/020 | Batch 0300/0469 | Cost: 0.0123\n",
|
||||||
|
"Epoch: 009/020 | Batch 0350/0469 | Cost: 0.0544\n",
|
||||||
|
"Epoch: 009/020 | Batch 0400/0469 | Cost: 0.0174\n",
|
||||||
|
"Epoch: 009/020 | Batch 0450/0469 | Cost: 0.0093\n",
|
||||||
|
"Epoch: 009/020 | Train: 98.930%\n",
|
||||||
|
"Time elapsed: 6.13 min\n",
|
||||||
|
"Epoch: 010/020 | Batch 0000/0469 | Cost: 0.0164\n",
|
||||||
|
"Epoch: 010/020 | Batch 0050/0469 | Cost: 0.0301\n",
|
||||||
|
"Epoch: 010/020 | Batch 0100/0469 | Cost: 0.0198\n",
|
||||||
|
"Epoch: 010/020 | Batch 0150/0469 | Cost: 0.0171\n",
|
||||||
|
"Epoch: 010/020 | Batch 0200/0469 | Cost: 0.1067\n",
|
||||||
|
"Epoch: 010/020 | Batch 0250/0469 | Cost: 0.0099\n",
|
||||||
|
"Epoch: 010/020 | Batch 0300/0469 | Cost: 0.0169\n",
|
||||||
|
"Epoch: 010/020 | Batch 0350/0469 | Cost: 0.0498\n",
|
||||||
|
"Epoch: 010/020 | Batch 0400/0469 | Cost: 0.0394\n",
|
||||||
|
"Epoch: 010/020 | Batch 0450/0469 | Cost: 0.0366\n",
|
||||||
|
"Epoch: 010/020 | Train: 99.385%\n",
|
||||||
|
"Time elapsed: 6.81 min\n",
|
||||||
|
"Epoch: 011/020 | Batch 0000/0469 | Cost: 0.0049\n",
|
||||||
|
"Epoch: 011/020 | Batch 0050/0469 | Cost: 0.0052\n",
|
||||||
|
"Epoch: 011/020 | Batch 0100/0469 | Cost: 0.0019\n",
|
||||||
|
"Epoch: 011/020 | Batch 0150/0469 | Cost: 0.0270\n",
|
||||||
|
"Epoch: 011/020 | Batch 0200/0469 | Cost: 0.0076\n",
|
||||||
|
"Epoch: 011/020 | Batch 0250/0469 | Cost: 0.0091\n",
|
||||||
|
"Epoch: 011/020 | Batch 0300/0469 | Cost: 0.0114\n",
|
||||||
|
"Epoch: 011/020 | Batch 0350/0469 | Cost: 0.0233\n",
|
||||||
|
"Epoch: 011/020 | Batch 0400/0469 | Cost: 0.0443\n",
|
||||||
|
"Epoch: 011/020 | Batch 0450/0469 | Cost: 0.0027\n",
|
||||||
|
"Epoch: 011/020 | Train: 99.693%\n",
|
||||||
|
"Time elapsed: 7.50 min\n",
|
||||||
|
"Epoch: 012/020 | Batch 0000/0469 | Cost: 0.0361\n",
|
||||||
|
"Epoch: 012/020 | Batch 0050/0469 | Cost: 0.0054\n",
|
||||||
|
"Epoch: 012/020 | Batch 0100/0469 | Cost: 0.0485\n",
|
||||||
|
"Epoch: 012/020 | Batch 0150/0469 | Cost: 0.0220\n",
|
||||||
|
"Epoch: 012/020 | Batch 0200/0469 | Cost: 0.0903\n",
|
||||||
|
"Epoch: 012/020 | Batch 0250/0469 | Cost: 0.0144\n",
|
||||||
|
"Epoch: 012/020 | Batch 0300/0469 | Cost: 0.0148\n",
|
||||||
|
"Epoch: 012/020 | Batch 0350/0469 | Cost: 0.0055\n",
|
||||||
|
"Epoch: 012/020 | Batch 0400/0469 | Cost: 0.0012\n",
|
||||||
|
"Epoch: 012/020 | Batch 0450/0469 | Cost: 0.0228\n",
|
||||||
|
"Epoch: 012/020 | Train: 99.530%\n",
|
||||||
|
"Time elapsed: 8.18 min\n",
|
||||||
|
"Epoch: 013/020 | Batch 0000/0469 | Cost: 0.0038\n",
|
||||||
|
"Epoch: 013/020 | Batch 0050/0469 | Cost: 0.0060\n",
|
||||||
|
"Epoch: 013/020 | Batch 0100/0469 | Cost: 0.0206\n",
|
||||||
|
"Epoch: 013/020 | Batch 0150/0469 | Cost: 0.0092\n",
|
||||||
|
"Epoch: 013/020 | Batch 0200/0469 | Cost: 0.0428\n",
|
||||||
|
"Epoch: 013/020 | Batch 0250/0469 | Cost: 0.0627\n",
|
||||||
|
"Epoch: 013/020 | Batch 0300/0469 | Cost: 0.0374\n",
|
||||||
|
"Epoch: 013/020 | Batch 0350/0469 | Cost: 0.0160\n",
|
||||||
|
"Epoch: 013/020 | Batch 0400/0469 | Cost: 0.0013\n",
|
||||||
|
"Epoch: 013/020 | Batch 0450/0469 | Cost: 0.0477\n",
|
||||||
|
"Epoch: 013/020 | Train: 99.625%\n",
|
||||||
|
"Time elapsed: 8.86 min\n",
|
||||||
|
"Epoch: 014/020 | Batch 0000/0469 | Cost: 0.0087\n",
|
||||||
|
"Epoch: 014/020 | Batch 0050/0469 | Cost: 0.0014\n",
|
||||||
|
"Epoch: 014/020 | Batch 0100/0469 | Cost: 0.0032\n",
|
||||||
|
"Epoch: 014/020 | Batch 0150/0469 | Cost: 0.0096\n",
|
||||||
|
"Epoch: 014/020 | Batch 0200/0469 | Cost: 0.0128\n",
|
||||||
|
"Epoch: 014/020 | Batch 0250/0469 | Cost: 0.0131\n",
|
||||||
|
"Epoch: 014/020 | Batch 0300/0469 | Cost: 0.0137\n",
|
||||||
|
"Epoch: 014/020 | Batch 0350/0469 | Cost: 0.0338\n",
|
||||||
|
"Epoch: 014/020 | Batch 0400/0469 | Cost: 0.0393\n",
|
||||||
|
"Epoch: 014/020 | Batch 0450/0469 | Cost: 0.0372\n",
|
||||||
|
"Epoch: 014/020 | Train: 99.483%\n",
|
||||||
|
"Time elapsed: 9.55 min\n",
|
||||||
|
"Epoch: 015/020 | Batch 0000/0469 | Cost: 0.0263\n",
|
||||||
|
"Epoch: 015/020 | Batch 0050/0469 | Cost: 0.0049\n",
|
||||||
|
"Epoch: 015/020 | Batch 0100/0469 | Cost: 0.0198\n",
|
||||||
|
"Epoch: 015/020 | Batch 0150/0469 | Cost: 0.0455\n",
|
||||||
|
"Epoch: 015/020 | Batch 0200/0469 | Cost: 0.0028\n",
|
||||||
|
"Epoch: 015/020 | Batch 0250/0469 | Cost: 0.0069\n",
|
||||||
|
"Epoch: 015/020 | Batch 0300/0469 | Cost: 0.0319\n",
|
||||||
|
"Epoch: 015/020 | Batch 0350/0469 | Cost: 0.0006\n",
|
||||||
|
"Epoch: 015/020 | Batch 0400/0469 | Cost: 0.0022\n",
|
||||||
|
"Epoch: 015/020 | Batch 0450/0469 | Cost: 0.0024\n",
|
||||||
|
"Epoch: 015/020 | Train: 99.795%\n",
|
||||||
|
"Time elapsed: 10.24 min\n",
|
||||||
|
"Epoch: 016/020 | Batch 0000/0469 | Cost: 0.0010\n",
|
||||||
|
"Epoch: 016/020 | Batch 0050/0469 | Cost: 0.0029\n",
|
||||||
|
"Epoch: 016/020 | Batch 0100/0469 | Cost: 0.0031\n",
|
||||||
|
"Epoch: 016/020 | Batch 0150/0469 | Cost: 0.0041\n",
|
||||||
|
"Epoch: 016/020 | Batch 0200/0469 | Cost: 0.0007\n",
|
||||||
|
"Epoch: 016/020 | Batch 0250/0469 | Cost: 0.0130\n",
|
||||||
|
"Epoch: 016/020 | Batch 0300/0469 | Cost: 0.0172\n",
|
||||||
|
"Epoch: 016/020 | Batch 0350/0469 | Cost: 0.0391\n",
|
||||||
|
"Epoch: 016/020 | Batch 0400/0469 | Cost: 0.0171\n",
|
||||||
|
"Epoch: 016/020 | Batch 0450/0469 | Cost: 0.0763\n",
|
||||||
|
"Epoch: 016/020 | Train: 99.533%\n",
|
||||||
|
"Time elapsed: 10.92 min\n",
|
||||||
|
"Epoch: 017/020 | Batch 0000/0469 | Cost: 0.0575\n",
|
||||||
|
"Epoch: 017/020 | Batch 0050/0469 | Cost: 0.0122\n",
|
||||||
|
"Epoch: 017/020 | Batch 0100/0469 | Cost: 0.0356\n",
|
||||||
|
"Epoch: 017/020 | Batch 0150/0469 | Cost: 0.0309\n",
|
||||||
|
"Epoch: 017/020 | Batch 0200/0469 | Cost: 0.0840\n",
|
||||||
|
"Epoch: 017/020 | Batch 0250/0469 | Cost: 0.0178\n",
|
||||||
|
"Epoch: 017/020 | Batch 0300/0469 | Cost: 0.0083\n",
|
||||||
|
"Epoch: 017/020 | Batch 0350/0469 | Cost: 0.0006\n",
|
||||||
|
"Epoch: 017/020 | Batch 0400/0469 | Cost: 0.0114\n",
|
||||||
|
"Epoch: 017/020 | Batch 0450/0469 | Cost: 0.0281\n",
|
||||||
|
"Epoch: 017/020 | Train: 99.777%\n",
|
||||||
|
"Time elapsed: 11.62 min\n",
|
||||||
|
"Epoch: 018/020 | Batch 0000/0469 | Cost: 0.0116\n",
|
||||||
|
"Epoch: 018/020 | Batch 0050/0469 | Cost: 0.0014\n",
|
||||||
|
"Epoch: 018/020 | Batch 0100/0469 | Cost: 0.0149\n",
|
||||||
|
"Epoch: 018/020 | Batch 0150/0469 | Cost: 0.0258\n",
|
||||||
|
"Epoch: 018/020 | Batch 0200/0469 | Cost: 0.0032\n",
|
||||||
|
"Epoch: 018/020 | Batch 0250/0469 | Cost: 0.0026\n",
|
||||||
|
"Epoch: 018/020 | Batch 0300/0469 | Cost: 0.0010\n",
|
||||||
|
"Epoch: 018/020 | Batch 0350/0469 | Cost: 0.0109\n",
|
||||||
|
"Epoch: 018/020 | Batch 0400/0469 | Cost: 0.0003\n",
|
||||||
|
"Epoch: 018/020 | Batch 0450/0469 | Cost: 0.0052\n",
|
||||||
|
"Epoch: 018/020 | Train: 99.540%\n",
|
||||||
|
"Time elapsed: 12.30 min\n",
|
||||||
|
"Epoch: 019/020 | Batch 0000/0469 | Cost: 0.0215\n",
|
||||||
|
"Epoch: 019/020 | Batch 0050/0469 | Cost: 0.0025\n",
|
||||||
|
"Epoch: 019/020 | Batch 0100/0469 | Cost: 0.0884\n",
|
||||||
|
"Epoch: 019/020 | Batch 0150/0469 | Cost: 0.0038\n",
|
||||||
|
"Epoch: 019/020 | Batch 0200/0469 | Cost: 0.0036\n",
|
||||||
|
"Epoch: 019/020 | Batch 0250/0469 | Cost: 0.0061\n",
|
||||||
|
"Epoch: 019/020 | Batch 0300/0469 | Cost: 0.0015\n",
|
||||||
|
"Epoch: 019/020 | Batch 0350/0469 | Cost: 0.0406\n",
|
||||||
|
"Epoch: 019/020 | Batch 0400/0469 | Cost: 0.1211\n",
|
||||||
|
"Epoch: 019/020 | Batch 0450/0469 | Cost: 0.0135\n",
|
||||||
|
"Epoch: 019/020 | Train: 99.617%\n",
|
||||||
|
"Time elapsed: 12.98 min\n",
|
||||||
|
"Epoch: 020/020 | Batch 0000/0469 | Cost: 0.0983\n",
|
||||||
|
"Epoch: 020/020 | Batch 0050/0469 | Cost: 0.0043\n",
|
||||||
|
"Epoch: 020/020 | Batch 0100/0469 | Cost: 0.0492\n",
|
||||||
|
"Epoch: 020/020 | Batch 0150/0469 | Cost: 0.0634\n",
|
||||||
|
"Epoch: 020/020 | Batch 0200/0469 | Cost: 0.0052\n",
|
||||||
|
"Epoch: 020/020 | Batch 0250/0469 | Cost: 0.0082\n",
|
||||||
|
"Epoch: 020/020 | Batch 0300/0469 | Cost: 0.0044\n",
|
||||||
|
"Epoch: 020/020 | Batch 0350/0469 | Cost: 0.0015\n",
|
||||||
|
"Epoch: 020/020 | Batch 0400/0469 | Cost: 0.0153\n",
|
||||||
|
"Epoch: 020/020 | Batch 0450/0469 | Cost: 0.0085\n",
|
||||||
|
"Epoch: 020/020 | Train: 99.685%\n",
|
||||||
|
"Time elapsed: 13.67 min\n",
|
||||||
|
"Total Training Time: 13.67 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader, device):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for i, (features, targets) in enumerate(data_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(NUM_EPOCHS):\n",
|
||||||
|
" \n",
|
||||||
|
" model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" \n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, NUM_EPOCHS, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Epoch: %03d/%03d | Train: %.3f%%' % (\n",
|
||||||
|
" epoch+1, NUM_EPOCHS, \n",
|
||||||
|
" compute_accuracy(model, train_loader, device=DEVICE)))\n",
|
||||||
|
" \n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "paaeEQHQj5xC"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 34
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 6514,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976895054,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "gzQMWKq5j5xE",
|
||||||
|
"outputId": "de7dc005-5eeb-4177-9f9f-d9b5d1358db9"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 98.39%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader, device=DEVICE)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADU5JREFUeJzt3W+IXfWdx/HPZ2OjwRZ1zGwc0ujEIuuouMkyxGDD0qXbYLUQ80DpKCWL0vRBlS32gX/2wUZBDMu2NQ+WwnQTE7Vru9DGRJC12bBiChocZVZNXXc0TklC/kxIMVaEavLdB3PSnercc6/337mT7/sFw9x7vufPl0M+Offe353zc0QIQD5/VnUDAKpB+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJHVONw+2cOHCGBwc7OYhgVQmJyd1/PhxN7JuS+G3fYOkTZLmSfrXiNhYtv7g4KDGxsZaOSSAEsPDww2v2/TLftvzJP2LpK9LukrSiO2rmt0fgO5q5T3/CklvR8T+iPiDpJ9JWtOetgB0WivhXyzpwIznB4tlf8L2ettjtsempqZaOByAdur4p/0RMRoRwxEx3N/f3+nDAWhQK+E/JGnJjOdfLJYBmANaCf/Lkq6wvdT2fEnflLSzPW0B6LSmh/oi4mPbd0l6TtNDfVsiYl/bOgPQUS2N80fEs5KebVMvALqIr/cCSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QVEuz9NqelPS+pFOSPo6I4XY0BaDzWgp/4W8i4ngb9gOgi3jZDyTVavhD0q9sv2J7fTsaAtAdrb7sXxURh2z/uaRdtv8nIl6YuULxn8J6Sbr00ktbPByAdmnpyh8Rh4rfxyRtl7RilnVGI2I4Iob7+/tbORyANmo6/LbPt/2FM48lrZb0RrsaA9BZrbzsXyRpu+0z+/m3iPiPtnQFoOOaDn9E7Jf0l23sBUAXMdQHJEX4gaQIP5AU4QeSIvxAUoQfSKodf9WXwksvvVSztmnTptJtFy9eXFpfsGBBaX3dunWl9b6+vqZqyI0rP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kxTh/g8rG2icmJjp67Icffri0fsEFF9SsrVy5st3tzBmDg4M1a/fff3/pthluOceVH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSYpy/QU8//XTN2vj4eOm2V199dWl93759pfW9e/eW1nfs2FGz9txzz5Vuu3Tp0tL6u+++W1pvxTnnlP/zGxgYKK0fOHCg6WOXfQdAku69996m9z1XcOUHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaTqjvPb3iLpG5KORcQ1xbI+ST+XNChpUtKtEfG7zrVZvaGhoaZqjbj22mtL6yMjI6X1jRs31qxNTk6WbltvnH///v2l9VbMnz+/tF5vnL9e71NTUzVrV155Zem2GTRy5d8q6YZPLLtP0u6IuELS7uI5gDmkbvgj4gVJJz6xeI2kbcXjbZJubnNfADqs2ff8iyLicPH4iKRFbeoHQJe0/IFfRISkqFW3vd72mO2xsvdgALqr2fAftT0gScXvY7VWjIjRiBiOiOH+/v4mDweg3ZoN/05JZ25nu05S7T8rA9CT6obf9lOSXpT0F7YP2r5T0kZJX7M9Ielvi+cA5pC64/wRUWuQ+att7gVNOu+882rWWh3PbvU7DK2odx+D48ePl9avu+66mrXVq1c31dPZhG/4AUkRfiApwg8kRfiBpAg/kBThB5Li1t2ozAcffFBaX7t2bWn99OnTpfVHH320Zm3BggWl22bAlR9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmKcH5XZunVraf3IkSOl9Ysvvri0ftlll33WllLhyg8kRfiBpAg/kBThB5Ii/EBShB9IivADSTHOj4565513atbuueeelvb94osvltYvueSSlvZ/tuPKDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJ1R3nt71F0jckHYuIa4plGyR9W9JUsdoDEfFsp5rE3PXMM8/UrH300Uel295yyy2l9csvv7ypnjCtkSv/Vkk3zLL8RxGxrPgh+MAcUzf8EfGCpBNd6AVAF7Xynv8u26/Z3mL7orZ1BKArmg3/jyV9SdIySYcl/aDWirbX2x6zPTY1NVVrNQBd1lT4I+JoRJyKiNOSfiJpRcm6oxExHBHD/f39zfYJoM2aCr/tgRlP10p6oz3tAOiWRob6npL0FUkLbR+U9I+SvmJ7maSQNCnpOx3sEUAH1A1/RIzMsnhzB3rBHFRvrH779u01a+eee27pto888khpfd68eaV1lOMbfkBShB9IivADSRF+ICnCDyRF+IGkuHU3WrJ5c/mo7549e2rWbrvtttJt+ZPdzuLKDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJMc6PUuPj46X1u+++u7R+4YUX1qw99NBDTfWE9uDKDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJMc6f3IcfflhaHxmZ7c7t/+/UqVOl9dtvv71mjb/XrxZXfiApwg8kRfiBpAg/kBThB5Ii/EBShB9Iqu44v+0lkh6XtEhSSBqNiE22+yT9XNKgpElJt0bE7zrXKppx+vTp0vpNN91UWn/rrbdK60NDQ6X1Bx98sLSO6jRy5f9Y0vcj4ipJKyV91/ZVku6TtDsirpC0u3gOYI6oG/6IOBwRrxaP35f0pqTFktZI2lastk3SzZ1qEkD7fab3/LYHJS2XtFfSoog4XJSOaPptAYA5ouHw2/68pF9I+l5EnJxZi4jQ9OcBs2233vaY7bGpqamWmgXQPg2F3/bnNB38n0bEL4vFR20PFPUBScdm2zYiRiNiOCKG+/v729EzgDaoG37blrRZ0psR8cMZpZ2S1hWP10na0f72AHRKI3/S+2VJ35L0uu0z93F+QNJGSf9u+05Jv5V0a2daRCtOnDhRWn/++edb2v8TTzxRWu/r62tp/+icuuGPiF9Lco3yV9vbDoBu4Rt+QFKEH0iK8ANJEX4gKcIPJEX4gaS4dfdZ4L333qtZW7lyZUv7fvLJJ0vry5cvb2n/qA5XfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IinH+s8Bjjz1Ws7Z///6W9r1q1arS+vS9XjAXceUHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQY558DJiYmSusbNmzoTiM4q3DlB5Ii/EBShB9IivADSRF+ICnCDyRF+IGk6o7z214i6XFJiySFpNGI2GR7g6RvS5oqVn0gIp7tVKOZ7dmzp7R+8uTJpvc9NDRUWl+wYEHT+0Zva+RLPh9L+n5EvGr7C5Jesb2rqP0oIv65c+0B6JS64Y+Iw5IOF4/ft/2mpMWdbgxAZ32m9/y2ByUtl7S3WHSX7ddsb7F9UY1t1tsesz02NTU12yoAKtBw+G1/XtIvJH0vIk5K+rGkL0lapulXBj+YbbuIGI2I4YgY7u/vb0PLANqhofDb/pymg//TiPilJEXE0Yg4FRGnJf1E0orOtQmg3eqG39O3Z90s6c2I+OGM5QMzVlsr6Y32twegUxr5tP/Lkr4l6XXb48WyBySN2F6m6eG/SUnf6UiHaMn1119fWt+1a1dpnaG+s1cjn/b/WtJsN2dnTB+Yw/iGH5AU4QeSIvxAUoQfSIrwA0kRfiApbt09B9xxxx0t1YHZcOUHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQcEd07mD0l6bczFi2UdLxrDXw2vdpbr/Yl0Vuz2tnbZRHR0P3yuhr+Tx3cHouI4coaKNGrvfVqXxK9Nauq3njZDyRF+IGkqg7/aMXHL9OrvfVqXxK9NauS3ip9zw+gOlVf+QFUpJLw277B9lu237Z9XxU91GJ70vbrtsdtj1Xcyxbbx2y/MWNZn+1dtieK37NOk1ZRbxtsHyrO3bjtGyvqbYnt/7L9G9v7bP99sbzSc1fSVyXnresv+23Pk/S/kr4m6aCklyWNRMRvutpIDbYnJQ1HROVjwrb/WtLvJT0eEdcUy/5J0omI2Fj8x3lRRNzbI71tkPT7qmduLiaUGZg5s7SkmyX9nSo8dyV93aoKzlsVV/4Vkt6OiP0R8QdJP5O0poI+el5EvCDpxCcWr5G0rXi8TdP/eLquRm89ISIOR8SrxeP3JZ2ZWbrSc1fSVyWqCP9iSQdmPD+o3pryOyT9yvYrttdX3cwsFhXTpkvSEUmLqmxmFnVnbu6mT8ws3TPnrpkZr9uND/w+bVVE/JWkr0v6bvHytifF9Hu2XhquaWjm5m6ZZWbpP6ry3DU743W7VRH+Q5KWzHj+xWJZT4iIQ8XvY5K2q/dmHz56ZpLU4vexivv5o16auXm2maXVA+eul2a8riL8L0u6wvZS2/MlfVPSzgr6+BTb5xcfxMj2+ZJWq/dmH94paV3xeJ2kHRX28id6ZebmWjNLq+Jz13MzXkdE138k3ajpT/zfkfQPVfRQo6/LJf138bOv6t4kPaXpl4EfafqzkTslXSxpt6QJSf8pqa+HentC0uuSXtN00AYq6m2Vpl/SvyZpvPi5sepzV9JXJeeNb/gBSfGBH5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpP4Pc0oGVHoLWbQAAAAASUVORK5CYII=\n",
|
||||||
|
"text/plain": [
|
||||||
|
"<Figure size 432x288 with 1 Axes>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"needs_background": "light"
|
||||||
|
},
|
||||||
|
"output_type": "display_data"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"for batch_idx, (features, targets) in enumerate(test_loader):\n",
|
||||||
|
"\n",
|
||||||
|
" features = features\n",
|
||||||
|
" targets = targets\n",
|
||||||
|
" break\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
"nhwc_img = np.transpose(features[0], axes=(1, 2, 0))\n",
|
||||||
|
"nhw_img = np.squeeze(nhwc_img.numpy(), axis=2)\n",
|
||||||
|
"plt.imshow(nhw_img, cmap='Greys');"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 11,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Probability 7 100.00%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"model.eval()\n",
|
||||||
|
"logits, probas = model(features.to(device)[0, None])\n",
|
||||||
|
"print('Probability 7 %.2f%%' % (probas[0][7]*100))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 12,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"numpy 1.15.4\n",
|
||||||
|
"pandas 0.23.4\n",
|
||||||
|
"torch 1.0.0\n",
|
||||||
|
"PIL.Image 5.3.0\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"accelerator": "GPU",
|
||||||
|
"colab": {
|
||||||
|
"collapsed_sections": [],
|
||||||
|
"default_view": {},
|
||||||
|
"name": "convnet-vgg16.ipynb",
|
||||||
|
"provenance": [],
|
||||||
|
"version": "0.3.2",
|
||||||
|
"views": {}
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.1"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": true,
|
||||||
|
"toc_position": {
|
||||||
|
"height": "calc(100% - 180px)",
|
||||||
|
"left": "10px",
|
||||||
|
"top": "150px",
|
||||||
|
"width": "371px"
|
||||||
|
},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
@@ -0,0 +1,639 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.6.8\n",
|
||||||
|
"IPython 7.2.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.0.0\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"- Runs on CPU or GPU (if available)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -Standardizing Images"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"This notebook provides an example for working with standardized images, that is, images where the image pixels in each image has mean zero and unit variance across the channel. \n",
|
||||||
|
"\n",
|
||||||
|
"The general equation for z-score standardization is computed as \n",
|
||||||
|
"\n",
|
||||||
|
"$$x' = \\frac{x_i - \\mu}{\\sigma}$$\n",
|
||||||
|
"\n",
|
||||||
|
"where $\\mu$ is the mean and $\\sigma$ is the standard deviation of the training set, respectively. Then $x_i'$ is the scaled feature feature value, and $x_i$ is the original feature value.\n",
|
||||||
|
"\n",
|
||||||
|
"I.e, for grayscale images, we would obtain 1 mean and 1 standard deviation. For RGB images (3 color channels), we would obtain 3 mean values and 3 standard deviations."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import time\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Settings and Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Device\n",
|
||||||
|
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"random_seed = 1\n",
|
||||||
|
"learning_rate = 0.05\n",
|
||||||
|
"num_epochs = 10\n",
|
||||||
|
"batch_size = 128\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"num_classes = 10"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Compute the Mean and Standard Deviation for Normalization"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"First, we need to determine the mean and standard deviation for each color channel in the training set. Since we assume the entire dataset does not fit into the computer memory all at once, we do this in an incremental fashion, as shown below."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Mean: tensor([0.1307])\n",
|
||||||
|
"Std Dev: tensor([0.3077])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##############################\n",
|
||||||
|
"### PRELIMINARY DATALOADER\n",
|
||||||
|
"##############################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"train_mean = []\n",
|
||||||
|
"train_std = []\n",
|
||||||
|
"\n",
|
||||||
|
"for i, image in enumerate(train_loader, 0):\n",
|
||||||
|
" numpy_image = image[0].numpy()\n",
|
||||||
|
" \n",
|
||||||
|
" batch_mean = np.mean(numpy_image, axis=(0, 2, 3))\n",
|
||||||
|
" batch_std = np.std(numpy_image, axis=(0, 2, 3))\n",
|
||||||
|
" \n",
|
||||||
|
" train_mean.append(batch_mean)\n",
|
||||||
|
" train_std.append(batch_std)\n",
|
||||||
|
"\n",
|
||||||
|
"train_mean = torch.tensor(np.mean(train_mean, axis=0))\n",
|
||||||
|
"train_std = torch.tensor(np.mean(train_std, axis=0))\n",
|
||||||
|
"\n",
|
||||||
|
"print('Mean:', train_mean)\n",
|
||||||
|
"print('Std Dev:', train_std)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"**Note that**\n",
|
||||||
|
"\n",
|
||||||
|
"- For RGB images (3 color channels), we would get 3 means and 3 standard deviations.\n",
|
||||||
|
"- The transforms.ToTensor() method converts images to [0, 1] range, which is why the mean and standard deviation values are below 1."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Standardized Dataset Loader"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Now we can use a custom transform function to standardize the dataset according the the mean and standard deviation we computed above."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"custom_transform = transforms.Compose([transforms.ToTensor(),\n",
|
||||||
|
" transforms.Normalize(mean=train_mean, std=train_std)])"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=custom_transform,\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.MNIST(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=custom_transform)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=False)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Check that the dataset can be loaded:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"For the given batch, check that the channel means and standard deviations are roughly 0 and 1, respectively:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Channel mean: tensor(0.0035)\n",
|
||||||
|
"Channel std: tensor(1.0037)\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"print('Channel mean:', torch.mean(images[:, 0, :, :]))\n",
|
||||||
|
"print('Channel std:', torch.std(images[:, 0, :, :]))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Model"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class ConvNet(torch.nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, num_classes):\n",
|
||||||
|
" super(ConvNet, self).__init__()\n",
|
||||||
|
" \n",
|
||||||
|
" # calculate same padding:\n",
|
||||||
|
" # (w - k + 2*p)/s + 1 = o\n",
|
||||||
|
" # => p = (s(o-1) - w + k)/2\n",
|
||||||
|
" \n",
|
||||||
|
" # 28x28x1 => 28x28x4\n",
|
||||||
|
" self.conv_1 = torch.nn.Conv2d(in_channels=1,\n",
|
||||||
|
" out_channels=4,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(28-1) - 28 + 3) / 2 = 1\n",
|
||||||
|
" # 28x28x4 => 14x14x4\n",
|
||||||
|
" self.pool_1 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2),\n",
|
||||||
|
" padding=0) # (2(14-1) - 28 + 2) = 0 \n",
|
||||||
|
" # 14x14x4 => 14x14x8\n",
|
||||||
|
" self.conv_2 = torch.nn.Conv2d(in_channels=4,\n",
|
||||||
|
" out_channels=8,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1) # (1(14-1) - 14 + 3) / 2 = 1 \n",
|
||||||
|
" # 14x14x8 => 7x7x8 \n",
|
||||||
|
" self.pool_2 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2),\n",
|
||||||
|
" padding=0) # (2(7-1) - 14 + 2) = 0\n",
|
||||||
|
" \n",
|
||||||
|
" self.linear_1 = torch.nn.Linear(7*7*8, num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
" out = self.conv_1(x)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" out = self.pool_1(out)\n",
|
||||||
|
"\n",
|
||||||
|
" out = self.conv_2(out)\n",
|
||||||
|
" out = F.relu(out)\n",
|
||||||
|
" out = self.pool_2(out)\n",
|
||||||
|
" \n",
|
||||||
|
" logits = self.linear_1(out.view(-1, 7*7*8))\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"torch.manual_seed(random_seed)\n",
|
||||||
|
"model = ConvNet(num_classes=num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
"model = model.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 000/469 | Cost: 2.3170\n",
|
||||||
|
"Epoch: 001/010 | Batch 050/469 | Cost: 0.9476\n",
|
||||||
|
"Epoch: 001/010 | Batch 100/469 | Cost: 0.4007\n",
|
||||||
|
"Epoch: 001/010 | Batch 150/469 | Cost: 0.2662\n",
|
||||||
|
"Epoch: 001/010 | Batch 200/469 | Cost: 0.3218\n",
|
||||||
|
"Epoch: 001/010 | Batch 250/469 | Cost: 0.2300\n",
|
||||||
|
"Epoch: 001/010 | Batch 300/469 | Cost: 0.1494\n",
|
||||||
|
"Epoch: 001/010 | Batch 350/469 | Cost: 0.1837\n",
|
||||||
|
"Epoch: 001/010 | Batch 400/469 | Cost: 0.2072\n",
|
||||||
|
"Epoch: 001/010 | Batch 450/469 | Cost: 0.1541\n",
|
||||||
|
"Epoch: 001/010 training accuracy: 95.68%\n",
|
||||||
|
"Time elapsed: 0.27 min\n",
|
||||||
|
"Epoch: 002/010 | Batch 000/469 | Cost: 0.1077\n",
|
||||||
|
"Epoch: 002/010 | Batch 050/469 | Cost: 0.1394\n",
|
||||||
|
"Epoch: 002/010 | Batch 100/469 | Cost: 0.1593\n",
|
||||||
|
"Epoch: 002/010 | Batch 150/469 | Cost: 0.2174\n",
|
||||||
|
"Epoch: 002/010 | Batch 200/469 | Cost: 0.1093\n",
|
||||||
|
"Epoch: 002/010 | Batch 250/469 | Cost: 0.1314\n",
|
||||||
|
"Epoch: 002/010 | Batch 300/469 | Cost: 0.1019\n",
|
||||||
|
"Epoch: 002/010 | Batch 350/469 | Cost: 0.1102\n",
|
||||||
|
"Epoch: 002/010 | Batch 400/469 | Cost: 0.1028\n",
|
||||||
|
"Epoch: 002/010 | Batch 450/469 | Cost: 0.0678\n",
|
||||||
|
"Epoch: 002/010 training accuracy: 96.50%\n",
|
||||||
|
"Time elapsed: 0.54 min\n",
|
||||||
|
"Epoch: 003/010 | Batch 000/469 | Cost: 0.0713\n",
|
||||||
|
"Epoch: 003/010 | Batch 050/469 | Cost: 0.1317\n",
|
||||||
|
"Epoch: 003/010 | Batch 100/469 | Cost: 0.1434\n",
|
||||||
|
"Epoch: 003/010 | Batch 150/469 | Cost: 0.0452\n",
|
||||||
|
"Epoch: 003/010 | Batch 200/469 | Cost: 0.0783\n",
|
||||||
|
"Epoch: 003/010 | Batch 250/469 | Cost: 0.2011\n",
|
||||||
|
"Epoch: 003/010 | Batch 300/469 | Cost: 0.1132\n",
|
||||||
|
"Epoch: 003/010 | Batch 350/469 | Cost: 0.0930\n",
|
||||||
|
"Epoch: 003/010 | Batch 400/469 | Cost: 0.0842\n",
|
||||||
|
"Epoch: 003/010 | Batch 450/469 | Cost: 0.1059\n",
|
||||||
|
"Epoch: 003/010 training accuracy: 96.39%\n",
|
||||||
|
"Time elapsed: 0.81 min\n",
|
||||||
|
"Epoch: 004/010 | Batch 000/469 | Cost: 0.1334\n",
|
||||||
|
"Epoch: 004/010 | Batch 050/469 | Cost: 0.1208\n",
|
||||||
|
"Epoch: 004/010 | Batch 100/469 | Cost: 0.0962\n",
|
||||||
|
"Epoch: 004/010 | Batch 150/469 | Cost: 0.1293\n",
|
||||||
|
"Epoch: 004/010 | Batch 200/469 | Cost: 0.0977\n",
|
||||||
|
"Epoch: 004/010 | Batch 250/469 | Cost: 0.0504\n",
|
||||||
|
"Epoch: 004/010 | Batch 300/469 | Cost: 0.0801\n",
|
||||||
|
"Epoch: 004/010 | Batch 350/469 | Cost: 0.0968\n",
|
||||||
|
"Epoch: 004/010 | Batch 400/469 | Cost: 0.2138\n",
|
||||||
|
"Epoch: 004/010 | Batch 450/469 | Cost: 0.0946\n",
|
||||||
|
"Epoch: 004/010 training accuracy: 97.52%\n",
|
||||||
|
"Time elapsed: 1.08 min\n",
|
||||||
|
"Epoch: 005/010 | Batch 000/469 | Cost: 0.0960\n",
|
||||||
|
"Epoch: 005/010 | Batch 050/469 | Cost: 0.0132\n",
|
||||||
|
"Epoch: 005/010 | Batch 100/469 | Cost: 0.1012\n",
|
||||||
|
"Epoch: 005/010 | Batch 150/469 | Cost: 0.0437\n",
|
||||||
|
"Epoch: 005/010 | Batch 200/469 | Cost: 0.0386\n",
|
||||||
|
"Epoch: 005/010 | Batch 250/469 | Cost: 0.0461\n",
|
||||||
|
"Epoch: 005/010 | Batch 300/469 | Cost: 0.1063\n",
|
||||||
|
"Epoch: 005/010 | Batch 350/469 | Cost: 0.0972\n",
|
||||||
|
"Epoch: 005/010 | Batch 400/469 | Cost: 0.0912\n",
|
||||||
|
"Epoch: 005/010 | Batch 450/469 | Cost: 0.0633\n",
|
||||||
|
"Epoch: 005/010 training accuracy: 97.84%\n",
|
||||||
|
"Time elapsed: 1.35 min\n",
|
||||||
|
"Epoch: 006/010 | Batch 000/469 | Cost: 0.0771\n",
|
||||||
|
"Epoch: 006/010 | Batch 050/469 | Cost: 0.0245\n",
|
||||||
|
"Epoch: 006/010 | Batch 100/469 | Cost: 0.0373\n",
|
||||||
|
"Epoch: 006/010 | Batch 150/469 | Cost: 0.0459\n",
|
||||||
|
"Epoch: 006/010 | Batch 200/469 | Cost: 0.1140\n",
|
||||||
|
"Epoch: 006/010 | Batch 250/469 | Cost: 0.0465\n",
|
||||||
|
"Epoch: 006/010 | Batch 300/469 | Cost: 0.0166\n",
|
||||||
|
"Epoch: 006/010 | Batch 350/469 | Cost: 0.0145\n",
|
||||||
|
"Epoch: 006/010 | Batch 400/469 | Cost: 0.0621\n",
|
||||||
|
"Epoch: 006/010 | Batch 450/469 | Cost: 0.0570\n",
|
||||||
|
"Epoch: 006/010 training accuracy: 97.94%\n",
|
||||||
|
"Time elapsed: 1.62 min\n",
|
||||||
|
"Epoch: 007/010 | Batch 000/469 | Cost: 0.1529\n",
|
||||||
|
"Epoch: 007/010 | Batch 050/469 | Cost: 0.0283\n",
|
||||||
|
"Epoch: 007/010 | Batch 100/469 | Cost: 0.0463\n",
|
||||||
|
"Epoch: 007/010 | Batch 150/469 | Cost: 0.0623\n",
|
||||||
|
"Epoch: 007/010 | Batch 200/469 | Cost: 0.0637\n",
|
||||||
|
"Epoch: 007/010 | Batch 250/469 | Cost: 0.0718\n",
|
||||||
|
"Epoch: 007/010 | Batch 300/469 | Cost: 0.0098\n",
|
||||||
|
"Epoch: 007/010 | Batch 350/469 | Cost: 0.0853\n",
|
||||||
|
"Epoch: 007/010 | Batch 400/469 | Cost: 0.0958\n",
|
||||||
|
"Epoch: 007/010 | Batch 450/469 | Cost: 0.0633\n",
|
||||||
|
"Epoch: 007/010 training accuracy: 98.17%\n",
|
||||||
|
"Time elapsed: 1.89 min\n",
|
||||||
|
"Epoch: 008/010 | Batch 000/469 | Cost: 0.1090\n",
|
||||||
|
"Epoch: 008/010 | Batch 050/469 | Cost: 0.0963\n",
|
||||||
|
"Epoch: 008/010 | Batch 100/469 | Cost: 0.1356\n",
|
||||||
|
"Epoch: 008/010 | Batch 150/469 | Cost: 0.0197\n",
|
||||||
|
"Epoch: 008/010 | Batch 200/469 | Cost: 0.0714\n",
|
||||||
|
"Epoch: 008/010 | Batch 250/469 | Cost: 0.0509\n",
|
||||||
|
"Epoch: 008/010 | Batch 300/469 | Cost: 0.0830\n",
|
||||||
|
"Epoch: 008/010 | Batch 350/469 | Cost: 0.0872\n",
|
||||||
|
"Epoch: 008/010 | Batch 400/469 | Cost: 0.0888\n",
|
||||||
|
"Epoch: 008/010 | Batch 450/469 | Cost: 0.1107\n",
|
||||||
|
"Epoch: 008/010 training accuracy: 97.96%\n",
|
||||||
|
"Time elapsed: 2.15 min\n",
|
||||||
|
"Epoch: 009/010 | Batch 000/469 | Cost: 0.0666\n",
|
||||||
|
"Epoch: 009/010 | Batch 050/469 | Cost: 0.0787\n",
|
||||||
|
"Epoch: 009/010 | Batch 100/469 | Cost: 0.1526\n",
|
||||||
|
"Epoch: 009/010 | Batch 150/469 | Cost: 0.0501\n",
|
||||||
|
"Epoch: 009/010 | Batch 200/469 | Cost: 0.0628\n",
|
||||||
|
"Epoch: 009/010 | Batch 250/469 | Cost: 0.1503\n",
|
||||||
|
"Epoch: 009/010 | Batch 300/469 | Cost: 0.0475\n",
|
||||||
|
"Epoch: 009/010 | Batch 350/469 | Cost: 0.0390\n",
|
||||||
|
"Epoch: 009/010 | Batch 400/469 | Cost: 0.0298\n",
|
||||||
|
"Epoch: 009/010 | Batch 450/469 | Cost: 0.0184\n",
|
||||||
|
"Epoch: 009/010 training accuracy: 98.25%\n",
|
||||||
|
"Time elapsed: 2.42 min\n",
|
||||||
|
"Epoch: 010/010 | Batch 000/469 | Cost: 0.0119\n",
|
||||||
|
"Epoch: 010/010 | Batch 050/469 | Cost: 0.0582\n",
|
||||||
|
"Epoch: 010/010 | Batch 100/469 | Cost: 0.0242\n",
|
||||||
|
"Epoch: 010/010 | Batch 150/469 | Cost: 0.0256\n",
|
||||||
|
"Epoch: 010/010 | Batch 200/469 | Cost: 0.0234\n",
|
||||||
|
"Epoch: 010/010 | Batch 250/469 | Cost: 0.0455\n",
|
||||||
|
"Epoch: 010/010 | Batch 300/469 | Cost: 0.0744\n",
|
||||||
|
"Epoch: 010/010 | Batch 350/469 | Cost: 0.1547\n",
|
||||||
|
"Epoch: 010/010 | Batch 400/469 | Cost: 0.0181\n",
|
||||||
|
"Epoch: 010/010 | Batch 450/469 | Cost: 0.0622\n",
|
||||||
|
"Epoch: 010/010 training accuracy: 98.18%\n",
|
||||||
|
"Time elapsed: 2.69 min\n",
|
||||||
|
"Total Training Time: 2.69 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader):\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for features, targets in data_loader:\n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(num_epochs):\n",
|
||||||
|
" model = model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(device)\n",
|
||||||
|
" targets = targets.to(device)\n",
|
||||||
|
"\n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, num_epochs, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
" \n",
|
||||||
|
" model = model.eval()\n",
|
||||||
|
" print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
|
||||||
|
" epoch+1, num_epochs, \n",
|
||||||
|
" compute_accuracy(model, train_loader)))\n",
|
||||||
|
" \n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 11,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 98.07%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 12,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"numpy 1.15.4\n",
|
||||||
|
"torch 1.0.0\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.1"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": false,
|
||||||
|
"toc_position": {},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,752 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"toc": true
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
|
||||||
|
"<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Model-Zoo----Convolutional-Neural-Network-(VGG16)\" data-toc-modified-id=\"Model-Zoo----Convolutional-Neural-Network-(VGG16)-1\"><span class=\"toc-item-num\">1 </span>Model Zoo -- Convolutional Neural Network (VGG16)</a></span><ul class=\"toc-item\"><li><span><a href=\"#Imports\" data-toc-modified-id=\"Imports-1.1\"><span class=\"toc-item-num\">1.1 </span>Imports</a></span></li><li><span><a href=\"#Settings-and-Dataset\" data-toc-modified-id=\"Settings-and-Dataset-1.2\"><span class=\"toc-item-num\">1.2 </span>Settings and Dataset</a></span></li><li><span><a href=\"#Model\" data-toc-modified-id=\"Model-1.3\"><span class=\"toc-item-num\">1.3 </span>Model</a></span></li><li><span><a href=\"#Training\" data-toc-modified-id=\"Training-1.4\"><span class=\"toc-item-num\">1.4 </span>Training</a></span></li><li><span><a href=\"#Evaluation\" data-toc-modified-id=\"Evaluation-1.5\"><span class=\"toc-item-num\">1.5 </span>Evaluation</a></span></li></ul></li></ul></div>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "UEBilEjLj5wY"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 119
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 536,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974472601,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "GOzuY8Yvj5wb",
|
||||||
|
"outputId": "c19362ce-f87a-4cc2-84cc-8d7b4b9e6007"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.7.3\n",
|
||||||
|
"IPython 7.6.1\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.3.0\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MEu9MiOxj5wk"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"- Runs on CPU (not recommended here) or GPU (if available)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "rH4XmErYj5wm"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- Convolutional Neural Network (VGG16)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MkoGLH_Tj5wn"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "ORj09gnrj5wp"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import time\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn as nn\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.backends.cudnn.deterministic = True"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "PvgJ_0i7j5wt"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Settings and Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 85
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 23936,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974497505,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "NnT0sZIwj5wu",
|
||||||
|
"outputId": "55aed925-d17e-4c6a-8c71-0d9b3bde5637"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Device: cuda:3\n",
|
||||||
|
"Files already downloaded and verified\n",
|
||||||
|
"Image batch dimensions: torch.Size([128, 3, 32, 32])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Device\n",
|
||||||
|
"DEVICE = torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||||
|
"print('Device:', DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"random_seed = 1\n",
|
||||||
|
"learning_rate = 0.001\n",
|
||||||
|
"num_epochs = 10\n",
|
||||||
|
"batch_size = 128\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"num_features = 784\n",
|
||||||
|
"num_classes = 10\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "I6hghKPxj5w0"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Model"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "_lza9t_uj5w1"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class VGG16(torch.nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, num_features, num_classes):\n",
|
||||||
|
" super(VGG16, self).__init__()\n",
|
||||||
|
" \n",
|
||||||
|
" # calculate same padding:\n",
|
||||||
|
" # (w - k + 2*p)/s + 1 = o\n",
|
||||||
|
" # => p = (s(o-1) - w + k)/2\n",
|
||||||
|
" \n",
|
||||||
|
" self.block_1 = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(in_channels=3,\n",
|
||||||
|
" out_channels=64,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" # (1(32-1)- 32 + 3)/2 = 1\n",
|
||||||
|
" padding=1), \n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.Conv2d(in_channels=64,\n",
|
||||||
|
" out_channels=64,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2))\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" self.block_2 = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(in_channels=64,\n",
|
||||||
|
" out_channels=128,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.Conv2d(in_channels=128,\n",
|
||||||
|
" out_channels=128,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2))\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" self.block_3 = nn.Sequential( \n",
|
||||||
|
" nn.Conv2d(in_channels=128,\n",
|
||||||
|
" out_channels=256,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.Conv2d(in_channels=256,\n",
|
||||||
|
" out_channels=256,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=256,\n",
|
||||||
|
" out_channels=256,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2))\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
" self.block_4 = nn.Sequential( \n",
|
||||||
|
" nn.Conv2d(in_channels=256,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2))\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" self.block_5 = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2)) \n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" self.classifier = nn.Sequential(\n",
|
||||||
|
" nn.Linear(512, 4096),\n",
|
||||||
|
" nn.ReLU(True),\n",
|
||||||
|
" #nn.Dropout(p=0.5),\n",
|
||||||
|
" nn.Linear(4096, 4096),\n",
|
||||||
|
" nn.ReLU(True),\n",
|
||||||
|
" #nn.Dropout(p=0.5),\n",
|
||||||
|
" nn.Linear(4096, num_classes),\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" for m in self.modules():\n",
|
||||||
|
" if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):\n",
|
||||||
|
" nn.init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='relu')\n",
|
||||||
|
" if m.bias is not None:\n",
|
||||||
|
" m.bias.detach().zero_()\n",
|
||||||
|
" \n",
|
||||||
|
" #self.avgpool = nn.AdaptiveAvgPool2d((7, 7))\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
"\n",
|
||||||
|
" x = self.block_1(x)\n",
|
||||||
|
" x = self.block_2(x)\n",
|
||||||
|
" x = self.block_3(x)\n",
|
||||||
|
" x = self.block_4(x)\n",
|
||||||
|
" x = self.block_5(x)\n",
|
||||||
|
" #x = self.avgpool(x)\n",
|
||||||
|
" x = x.view(x.size(0), -1)\n",
|
||||||
|
" logits = self.classifier(x)\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
"\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"torch.manual_seed(random_seed)\n",
|
||||||
|
"model = VGG16(num_features=num_features,\n",
|
||||||
|
" num_classes=num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
"model = model.to(DEVICE)\n",
|
||||||
|
"optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "RAodboScj5w6"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 1547
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 2384585,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976888520,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "Dzh3ROmRj5w7",
|
||||||
|
"outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/010 | Batch 0000/0391 | Cost: 2.4443\n",
|
||||||
|
"Epoch: 001/010 | Batch 0050/0391 | Cost: 2.2800\n",
|
||||||
|
"Epoch: 001/010 | Batch 0100/0391 | Cost: 2.1115\n",
|
||||||
|
"Epoch: 001/010 | Batch 0150/0391 | Cost: 1.9596\n",
|
||||||
|
"Epoch: 001/010 | Batch 0200/0391 | Cost: 2.0510\n",
|
||||||
|
"Epoch: 001/010 | Batch 0250/0391 | Cost: 1.8116\n",
|
||||||
|
"Epoch: 001/010 | Batch 0300/0391 | Cost: 1.8209\n",
|
||||||
|
"Epoch: 001/010 | Batch 0350/0391 | Cost: 1.7161\n",
|
||||||
|
"Epoch: 001/010 | Train: 36.232% | Loss: 1.600\n",
|
||||||
|
"Time elapsed: 0.59 min\n",
|
||||||
|
"Epoch: 002/010 | Batch 0000/0391 | Cost: 1.6484\n",
|
||||||
|
"Epoch: 002/010 | Batch 0050/0391 | Cost: 1.8022\n",
|
||||||
|
"Epoch: 002/010 | Batch 0100/0391 | Cost: 1.6488\n",
|
||||||
|
"Epoch: 002/010 | Batch 0150/0391 | Cost: 1.5652\n",
|
||||||
|
"Epoch: 002/010 | Batch 0200/0391 | Cost: 1.3576\n",
|
||||||
|
"Epoch: 002/010 | Batch 0250/0391 | Cost: 1.4434\n",
|
||||||
|
"Epoch: 002/010 | Batch 0300/0391 | Cost: 1.3092\n",
|
||||||
|
"Epoch: 002/010 | Batch 0350/0391 | Cost: 1.1899\n",
|
||||||
|
"Epoch: 002/010 | Train: 55.148% | Loss: 1.229\n",
|
||||||
|
"Time elapsed: 1.17 min\n",
|
||||||
|
"Epoch: 003/010 | Batch 0000/0391 | Cost: 1.1967\n",
|
||||||
|
"Epoch: 003/010 | Batch 0050/0391 | Cost: 1.1714\n",
|
||||||
|
"Epoch: 003/010 | Batch 0100/0391 | Cost: 1.3094\n",
|
||||||
|
"Epoch: 003/010 | Batch 0150/0391 | Cost: 1.2088\n",
|
||||||
|
"Epoch: 003/010 | Batch 0200/0391 | Cost: 1.2108\n",
|
||||||
|
"Epoch: 003/010 | Batch 0250/0391 | Cost: 0.9571\n",
|
||||||
|
"Epoch: 003/010 | Batch 0300/0391 | Cost: 1.0708\n",
|
||||||
|
"Epoch: 003/010 | Batch 0350/0391 | Cost: 0.9150\n",
|
||||||
|
"Epoch: 003/010 | Train: 67.218% | Loss: 0.938\n",
|
||||||
|
"Time elapsed: 1.77 min\n",
|
||||||
|
"Epoch: 004/010 | Batch 0000/0391 | Cost: 0.8263\n",
|
||||||
|
"Epoch: 004/010 | Batch 0050/0391 | Cost: 0.9029\n",
|
||||||
|
"Epoch: 004/010 | Batch 0100/0391 | Cost: 1.1502\n",
|
||||||
|
"Epoch: 004/010 | Batch 0150/0391 | Cost: 0.9787\n",
|
||||||
|
"Epoch: 004/010 | Batch 0200/0391 | Cost: 0.8300\n",
|
||||||
|
"Epoch: 004/010 | Batch 0250/0391 | Cost: 1.0111\n",
|
||||||
|
"Epoch: 004/010 | Batch 0300/0391 | Cost: 0.9322\n",
|
||||||
|
"Epoch: 004/010 | Batch 0350/0391 | Cost: 0.9228\n",
|
||||||
|
"Epoch: 004/010 | Train: 70.676% | Loss: 0.852\n",
|
||||||
|
"Time elapsed: 2.36 min\n",
|
||||||
|
"Epoch: 005/010 | Batch 0000/0391 | Cost: 0.8467\n",
|
||||||
|
"Epoch: 005/010 | Batch 0050/0391 | Cost: 0.6927\n",
|
||||||
|
"Epoch: 005/010 | Batch 0100/0391 | Cost: 0.8755\n",
|
||||||
|
"Epoch: 005/010 | Batch 0150/0391 | Cost: 0.8654\n",
|
||||||
|
"Epoch: 005/010 | Batch 0200/0391 | Cost: 0.8372\n",
|
||||||
|
"Epoch: 005/010 | Batch 0250/0391 | Cost: 0.6731\n",
|
||||||
|
"Epoch: 005/010 | Batch 0300/0391 | Cost: 0.6184\n",
|
||||||
|
"Epoch: 005/010 | Batch 0350/0391 | Cost: 0.6990\n",
|
||||||
|
"Epoch: 005/010 | Train: 73.508% | Loss: 0.764\n",
|
||||||
|
"Time elapsed: 2.96 min\n",
|
||||||
|
"Epoch: 006/010 | Batch 0000/0391 | Cost: 0.7755\n",
|
||||||
|
"Epoch: 006/010 | Batch 0050/0391 | Cost: 0.5677\n",
|
||||||
|
"Epoch: 006/010 | Batch 0100/0391 | Cost: 0.8109\n",
|
||||||
|
"Epoch: 006/010 | Batch 0150/0391 | Cost: 0.5656\n",
|
||||||
|
"Epoch: 006/010 | Batch 0200/0391 | Cost: 0.5641\n",
|
||||||
|
"Epoch: 006/010 | Batch 0250/0391 | Cost: 0.8524\n",
|
||||||
|
"Epoch: 006/010 | Batch 0300/0391 | Cost: 0.8494\n",
|
||||||
|
"Epoch: 006/010 | Batch 0350/0391 | Cost: 0.6887\n",
|
||||||
|
"Epoch: 006/010 | Train: 81.112% | Loss: 0.559\n",
|
||||||
|
"Time elapsed: 3.56 min\n",
|
||||||
|
"Epoch: 007/010 | Batch 0000/0391 | Cost: 0.5640\n",
|
||||||
|
"Epoch: 007/010 | Batch 0050/0391 | Cost: 0.5809\n",
|
||||||
|
"Epoch: 007/010 | Batch 0100/0391 | Cost: 0.7346\n",
|
||||||
|
"Epoch: 007/010 | Batch 0150/0391 | Cost: 0.6881\n",
|
||||||
|
"Epoch: 007/010 | Batch 0200/0391 | Cost: 0.4644\n",
|
||||||
|
"Epoch: 007/010 | Batch 0250/0391 | Cost: 0.5216\n",
|
||||||
|
"Epoch: 007/010 | Batch 0300/0391 | Cost: 0.6380\n",
|
||||||
|
"Epoch: 007/010 | Batch 0350/0391 | Cost: 0.5521\n",
|
||||||
|
"Epoch: 007/010 | Train: 84.438% | Loss: 0.460\n",
|
||||||
|
"Time elapsed: 4.15 min\n",
|
||||||
|
"Epoch: 008/010 | Batch 0000/0391 | Cost: 0.4843\n",
|
||||||
|
"Epoch: 008/010 | Batch 0050/0391 | Cost: 0.5226\n",
|
||||||
|
"Epoch: 008/010 | Batch 0100/0391 | Cost: 0.3682\n",
|
||||||
|
"Epoch: 008/010 | Batch 0150/0391 | Cost: 0.4941\n",
|
||||||
|
"Epoch: 008/010 | Batch 0200/0391 | Cost: 0.5376\n",
|
||||||
|
"Epoch: 008/010 | Batch 0250/0391 | Cost: 0.4707\n",
|
||||||
|
"Epoch: 008/010 | Batch 0300/0391 | Cost: 0.5882\n",
|
||||||
|
"Epoch: 008/010 | Batch 0350/0391 | Cost: 0.5355\n",
|
||||||
|
"Epoch: 008/010 | Train: 85.186% | Loss: 0.442\n",
|
||||||
|
"Time elapsed: 4.76 min\n",
|
||||||
|
"Epoch: 009/010 | Batch 0000/0391 | Cost: 0.4174\n",
|
||||||
|
"Epoch: 009/010 | Batch 0050/0391 | Cost: 0.4007\n",
|
||||||
|
"Epoch: 009/010 | Batch 0100/0391 | Cost: 0.4133\n",
|
||||||
|
"Epoch: 009/010 | Batch 0150/0391 | Cost: 0.6204\n",
|
||||||
|
"Epoch: 009/010 | Batch 0200/0391 | Cost: 0.3827\n",
|
||||||
|
"Epoch: 009/010 | Batch 0250/0391 | Cost: 0.4509\n",
|
||||||
|
"Epoch: 009/010 | Batch 0300/0391 | Cost: 0.4461\n",
|
||||||
|
"Epoch: 009/010 | Batch 0350/0391 | Cost: 0.4692\n",
|
||||||
|
"Epoch: 009/010 | Train: 89.038% | Loss: 0.328\n",
|
||||||
|
"Time elapsed: 5.36 min\n",
|
||||||
|
"Epoch: 010/010 | Batch 0000/0391 | Cost: 0.2964\n",
|
||||||
|
"Epoch: 010/010 | Batch 0050/0391 | Cost: 0.2739\n",
|
||||||
|
"Epoch: 010/010 | Batch 0100/0391 | Cost: 0.3803\n",
|
||||||
|
"Epoch: 010/010 | Batch 0150/0391 | Cost: 0.3981\n",
|
||||||
|
"Epoch: 010/010 | Batch 0200/0391 | Cost: 0.3877\n",
|
||||||
|
"Epoch: 010/010 | Batch 0250/0391 | Cost: 0.4213\n",
|
||||||
|
"Epoch: 010/010 | Batch 0300/0391 | Cost: 0.5088\n",
|
||||||
|
"Epoch: 010/010 | Batch 0350/0391 | Cost: 0.4598\n",
|
||||||
|
"Epoch: 010/010 | Train: 89.668% | Loss: 0.321\n",
|
||||||
|
"Time elapsed: 5.96 min\n",
|
||||||
|
"Total Training Time: 5.96 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader):\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for i, (features, targets) in enumerate(data_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def compute_epoch_loss(model, data_loader):\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" curr_loss, num_examples = 0., 0\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" for features, targets in data_loader:\n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" loss = F.cross_entropy(logits, targets, reduction='sum')\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" curr_loss += loss\n",
|
||||||
|
"\n",
|
||||||
|
" curr_loss = curr_loss / num_examples\n",
|
||||||
|
" return curr_loss\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(num_epochs):\n",
|
||||||
|
" \n",
|
||||||
|
" model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" \n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, num_epochs, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
"\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Epoch: %03d/%03d | Train: %.3f%% | Loss: %.3f' % (\n",
|
||||||
|
" epoch+1, num_epochs, \n",
|
||||||
|
" compute_accuracy(model, train_loader),\n",
|
||||||
|
" compute_epoch_loss(model, train_loader)))\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "paaeEQHQj5xC"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 34
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 6514,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976895054,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "gzQMWKq5j5xE",
|
||||||
|
"outputId": "de7dc005-5eeb-4177-9f9f-d9b5d1358db9"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 76.31%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"torchvision 0.4.1a0+d94043a\n",
|
||||||
|
"numpy 1.16.4\n",
|
||||||
|
"torch 1.3.0\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"accelerator": "GPU",
|
||||||
|
"colab": {
|
||||||
|
"collapsed_sections": [],
|
||||||
|
"default_view": {},
|
||||||
|
"name": "convnet-vgg16.ipynb",
|
||||||
|
"provenance": [],
|
||||||
|
"version": "0.3.2",
|
||||||
|
"views": {}
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.3"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": true,
|
||||||
|
"toc_position": {
|
||||||
|
"height": "calc(100% - 180px)",
|
||||||
|
"left": "10px",
|
||||||
|
"top": "150px",
|
||||||
|
"width": "371px"
|
||||||
|
},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
@@ -0,0 +1,873 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "UEBilEjLj5wY"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 119
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 536,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974472601,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "GOzuY8Yvj5wb",
|
||||||
|
"outputId": "c19362ce-f87a-4cc2-84cc-8d7b4b9e6007"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sebastian Raschka \n",
|
||||||
|
"\n",
|
||||||
|
"CPython 3.6.8\n",
|
||||||
|
"IPython 7.2.0\n",
|
||||||
|
"\n",
|
||||||
|
"torch 1.0.1.post2\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MEu9MiOxj5wk"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"- Runs on CPU (not recommended here) or GPU (if available)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "rH4XmErYj5wm"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"# Model Zoo -- Convolutional Neural Network (VGG19 Architecture)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Implementation of the VGG-19 architecture on Cifar10. \n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"Reference for VGG-19:\n",
|
||||||
|
" \n",
|
||||||
|
"- Simonyan, K., & Zisserman, A. (2014). Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556.\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"The following table (taken from Simonyan & Zisserman referenced above) summarizes the VGG19 architecture:\n",
|
||||||
|
"\n",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MkoGLH_Tj5wn"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Imports"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "ORj09gnrj5wp"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import time\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import torch.nn as nn\n",
|
||||||
|
"import torch.nn.functional as F\n",
|
||||||
|
"from torchvision import datasets\n",
|
||||||
|
"from torchvision import transforms\n",
|
||||||
|
"from torch.utils.data import DataLoader"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "PvgJ_0i7j5wt"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Settings and Dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 85
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 23936,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524974497505,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "NnT0sZIwj5wu",
|
||||||
|
"outputId": "55aed925-d17e-4c6a-8c71-0d9b3bde5637"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Device: cuda:0\n",
|
||||||
|
"Files already downloaded and verified\n",
|
||||||
|
"Image batch dimensions: torch.Size([128, 3, 32, 32])\n",
|
||||||
|
"Image label dimensions: torch.Size([128])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### SETTINGS\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Device\n",
|
||||||
|
"DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||||
|
"print('Device:', DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
"# Hyperparameters\n",
|
||||||
|
"random_seed = 1\n",
|
||||||
|
"learning_rate = 0.001\n",
|
||||||
|
"num_epochs = 20\n",
|
||||||
|
"batch_size = 128\n",
|
||||||
|
"\n",
|
||||||
|
"# Architecture\n",
|
||||||
|
"num_features = 784\n",
|
||||||
|
"num_classes = 10\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"##########################\n",
|
||||||
|
"### MNIST DATASET\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"# Note transforms.ToTensor() scales input images\n",
|
||||||
|
"# to 0-1 range\n",
|
||||||
|
"train_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=True, \n",
|
||||||
|
" transform=transforms.ToTensor(),\n",
|
||||||
|
" download=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_dataset = datasets.CIFAR10(root='data', \n",
|
||||||
|
" train=False, \n",
|
||||||
|
" transform=transforms.ToTensor())\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=True)\n",
|
||||||
|
"\n",
|
||||||
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
||||||
|
" batch_size=batch_size, \n",
|
||||||
|
" shuffle=False)\n",
|
||||||
|
"\n",
|
||||||
|
"# Checking the dataset\n",
|
||||||
|
"for images, labels in train_loader: \n",
|
||||||
|
" print('Image batch dimensions:', images.shape)\n",
|
||||||
|
" print('Image label dimensions:', labels.shape)\n",
|
||||||
|
" break"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "I6hghKPxj5w0"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Model"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "_lza9t_uj5w1"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"##########################\n",
|
||||||
|
"### MODEL\n",
|
||||||
|
"##########################\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"class VGG16(torch.nn.Module):\n",
|
||||||
|
"\n",
|
||||||
|
" def __init__(self, num_features, num_classes):\n",
|
||||||
|
" super(VGG16, self).__init__()\n",
|
||||||
|
" \n",
|
||||||
|
" # calculate same padding:\n",
|
||||||
|
" # (w - k + 2*p)/s + 1 = o\n",
|
||||||
|
" # => p = (s(o-1) - w + k)/2\n",
|
||||||
|
" \n",
|
||||||
|
" self.block_1 = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(in_channels=3,\n",
|
||||||
|
" out_channels=64,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" # (1(32-1)- 32 + 3)/2 = 1\n",
|
||||||
|
" padding=1), \n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.Conv2d(in_channels=64,\n",
|
||||||
|
" out_channels=64,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2))\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" self.block_2 = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(in_channels=64,\n",
|
||||||
|
" out_channels=128,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.Conv2d(in_channels=128,\n",
|
||||||
|
" out_channels=128,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2))\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" self.block_3 = nn.Sequential( \n",
|
||||||
|
" nn.Conv2d(in_channels=128,\n",
|
||||||
|
" out_channels=256,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.Conv2d(in_channels=256,\n",
|
||||||
|
" out_channels=256,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=256,\n",
|
||||||
|
" out_channels=256,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.Conv2d(in_channels=256,\n",
|
||||||
|
" out_channels=256,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2))\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
" self.block_4 = nn.Sequential( \n",
|
||||||
|
" nn.Conv2d(in_channels=256,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2))\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" self.block_5 = nn.Sequential(\n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(),\n",
|
||||||
|
" nn.Conv2d(in_channels=512,\n",
|
||||||
|
" out_channels=512,\n",
|
||||||
|
" kernel_size=(3, 3),\n",
|
||||||
|
" stride=(1, 1),\n",
|
||||||
|
" padding=1),\n",
|
||||||
|
" nn.ReLU(), \n",
|
||||||
|
" nn.MaxPool2d(kernel_size=(2, 2),\n",
|
||||||
|
" stride=(2, 2)) \n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" self.classifier = nn.Sequential(\n",
|
||||||
|
" nn.Linear(512, 4096),\n",
|
||||||
|
" nn.ReLU(True),\n",
|
||||||
|
" nn.Linear(4096, 4096),\n",
|
||||||
|
" nn.ReLU(True),\n",
|
||||||
|
" nn.Linear(4096, num_classes)\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
" for m in self.modules():\n",
|
||||||
|
" if isinstance(m, torch.nn.Conv2d):\n",
|
||||||
|
" #n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
|
||||||
|
" #m.weight.data.normal_(0, np.sqrt(2. / n))\n",
|
||||||
|
" m.weight.detach().normal_(0, 0.05)\n",
|
||||||
|
" if m.bias is not None:\n",
|
||||||
|
" m.bias.detach().zero_()\n",
|
||||||
|
" elif isinstance(m, torch.nn.Linear):\n",
|
||||||
|
" m.weight.detach().normal_(0, 0.05)\n",
|
||||||
|
" m.bias.detach().detach().zero_()\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
" def forward(self, x):\n",
|
||||||
|
"\n",
|
||||||
|
" x = self.block_1(x)\n",
|
||||||
|
" x = self.block_2(x)\n",
|
||||||
|
" x = self.block_3(x)\n",
|
||||||
|
" x = self.block_4(x)\n",
|
||||||
|
" x = self.block_5(x)\n",
|
||||||
|
" logits = self.classifier(x.view(-1, 512))\n",
|
||||||
|
" probas = F.softmax(logits, dim=1)\n",
|
||||||
|
"\n",
|
||||||
|
" return logits, probas\n",
|
||||||
|
"\n",
|
||||||
|
" \n",
|
||||||
|
"torch.manual_seed(random_seed)\n",
|
||||||
|
"model = VGG16(num_features=num_features,\n",
|
||||||
|
" num_classes=num_classes)\n",
|
||||||
|
"\n",
|
||||||
|
"model = model.to(DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
"optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "RAodboScj5w6"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Training"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 1547
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 2384585,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976888520,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "Dzh3ROmRj5w7",
|
||||||
|
"outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch: 001/020 | Batch 0000/0391 | Cost: 1061.4152\n",
|
||||||
|
"Epoch: 001/020 | Batch 0050/0391 | Cost: 2.3018\n",
|
||||||
|
"Epoch: 001/020 | Batch 0100/0391 | Cost: 2.0600\n",
|
||||||
|
"Epoch: 001/020 | Batch 0150/0391 | Cost: 1.9973\n",
|
||||||
|
"Epoch: 001/020 | Batch 0200/0391 | Cost: 1.8176\n",
|
||||||
|
"Epoch: 001/020 | Batch 0250/0391 | Cost: 1.8368\n",
|
||||||
|
"Epoch: 001/020 | Batch 0300/0391 | Cost: 1.7213\n",
|
||||||
|
"Epoch: 001/020 | Batch 0350/0391 | Cost: 1.7154\n",
|
||||||
|
"Epoch: 001/020 | Train: 35.478% | Loss: 1.685\n",
|
||||||
|
"Time elapsed: 1.02 min\n",
|
||||||
|
"Epoch: 002/020 | Batch 0000/0391 | Cost: 1.7648\n",
|
||||||
|
"Epoch: 002/020 | Batch 0050/0391 | Cost: 1.7050\n",
|
||||||
|
"Epoch: 002/020 | Batch 0100/0391 | Cost: 1.5464\n",
|
||||||
|
"Epoch: 002/020 | Batch 0150/0391 | Cost: 1.6054\n",
|
||||||
|
"Epoch: 002/020 | Batch 0200/0391 | Cost: 1.4430\n",
|
||||||
|
"Epoch: 002/020 | Batch 0250/0391 | Cost: 1.4253\n",
|
||||||
|
"Epoch: 002/020 | Batch 0300/0391 | Cost: 1.5701\n",
|
||||||
|
"Epoch: 002/020 | Batch 0350/0391 | Cost: 1.4163\n",
|
||||||
|
"Epoch: 002/020 | Train: 44.042% | Loss: 1.531\n",
|
||||||
|
"Time elapsed: 2.07 min\n",
|
||||||
|
"Epoch: 003/020 | Batch 0000/0391 | Cost: 1.5172\n",
|
||||||
|
"Epoch: 003/020 | Batch 0050/0391 | Cost: 1.1992\n",
|
||||||
|
"Epoch: 003/020 | Batch 0100/0391 | Cost: 1.2846\n",
|
||||||
|
"Epoch: 003/020 | Batch 0150/0391 | Cost: 1.4088\n",
|
||||||
|
"Epoch: 003/020 | Batch 0200/0391 | Cost: 1.4853\n",
|
||||||
|
"Epoch: 003/020 | Batch 0250/0391 | Cost: 1.3923\n",
|
||||||
|
"Epoch: 003/020 | Batch 0300/0391 | Cost: 1.3268\n",
|
||||||
|
"Epoch: 003/020 | Batch 0350/0391 | Cost: 1.3162\n",
|
||||||
|
"Epoch: 003/020 | Train: 55.596% | Loss: 1.223\n",
|
||||||
|
"Time elapsed: 3.10 min\n",
|
||||||
|
"Epoch: 004/020 | Batch 0000/0391 | Cost: 1.2210\n",
|
||||||
|
"Epoch: 004/020 | Batch 0050/0391 | Cost: 1.2594\n",
|
||||||
|
"Epoch: 004/020 | Batch 0100/0391 | Cost: 1.2881\n",
|
||||||
|
"Epoch: 004/020 | Batch 0150/0391 | Cost: 1.0182\n",
|
||||||
|
"Epoch: 004/020 | Batch 0200/0391 | Cost: 1.1256\n",
|
||||||
|
"Epoch: 004/020 | Batch 0250/0391 | Cost: 1.1048\n",
|
||||||
|
"Epoch: 004/020 | Batch 0300/0391 | Cost: 1.1812\n",
|
||||||
|
"Epoch: 004/020 | Batch 0350/0391 | Cost: 1.1685\n",
|
||||||
|
"Epoch: 004/020 | Train: 57.594% | Loss: 1.178\n",
|
||||||
|
"Time elapsed: 4.13 min\n",
|
||||||
|
"Epoch: 005/020 | Batch 0000/0391 | Cost: 1.1298\n",
|
||||||
|
"Epoch: 005/020 | Batch 0050/0391 | Cost: 0.9705\n",
|
||||||
|
"Epoch: 005/020 | Batch 0100/0391 | Cost: 0.9255\n",
|
||||||
|
"Epoch: 005/020 | Batch 0150/0391 | Cost: 1.3610\n",
|
||||||
|
"Epoch: 005/020 | Batch 0200/0391 | Cost: 0.9720\n",
|
||||||
|
"Epoch: 005/020 | Batch 0250/0391 | Cost: 1.0088\n",
|
||||||
|
"Epoch: 005/020 | Batch 0300/0391 | Cost: 0.9998\n",
|
||||||
|
"Epoch: 005/020 | Batch 0350/0391 | Cost: 1.1961\n",
|
||||||
|
"Epoch: 005/020 | Train: 63.570% | Loss: 1.003\n",
|
||||||
|
"Time elapsed: 5.17 min\n",
|
||||||
|
"Epoch: 006/020 | Batch 0000/0391 | Cost: 0.8837\n",
|
||||||
|
"Epoch: 006/020 | Batch 0050/0391 | Cost: 0.9184\n",
|
||||||
|
"Epoch: 006/020 | Batch 0100/0391 | Cost: 0.8568\n",
|
||||||
|
"Epoch: 006/020 | Batch 0150/0391 | Cost: 1.0788\n",
|
||||||
|
"Epoch: 006/020 | Batch 0200/0391 | Cost: 1.0365\n",
|
||||||
|
"Epoch: 006/020 | Batch 0250/0391 | Cost: 0.8714\n",
|
||||||
|
"Epoch: 006/020 | Batch 0300/0391 | Cost: 1.0370\n",
|
||||||
|
"Epoch: 006/020 | Batch 0350/0391 | Cost: 1.0536\n",
|
||||||
|
"Epoch: 006/020 | Train: 68.390% | Loss: 0.880\n",
|
||||||
|
"Time elapsed: 6.20 min\n",
|
||||||
|
"Epoch: 007/020 | Batch 0000/0391 | Cost: 1.0297\n",
|
||||||
|
"Epoch: 007/020 | Batch 0050/0391 | Cost: 0.8801\n",
|
||||||
|
"Epoch: 007/020 | Batch 0100/0391 | Cost: 0.9652\n",
|
||||||
|
"Epoch: 007/020 | Batch 0150/0391 | Cost: 1.1417\n",
|
||||||
|
"Epoch: 007/020 | Batch 0200/0391 | Cost: 0.8851\n",
|
||||||
|
"Epoch: 007/020 | Batch 0250/0391 | Cost: 0.9499\n",
|
||||||
|
"Epoch: 007/020 | Batch 0300/0391 | Cost: 0.9416\n",
|
||||||
|
"Epoch: 007/020 | Batch 0350/0391 | Cost: 0.9220\n",
|
||||||
|
"Epoch: 007/020 | Train: 68.740% | Loss: 0.872\n",
|
||||||
|
"Time elapsed: 7.24 min\n",
|
||||||
|
"Epoch: 008/020 | Batch 0000/0391 | Cost: 1.0054\n",
|
||||||
|
"Epoch: 008/020 | Batch 0050/0391 | Cost: 0.8184\n",
|
||||||
|
"Epoch: 008/020 | Batch 0100/0391 | Cost: 0.8955\n",
|
||||||
|
"Epoch: 008/020 | Batch 0150/0391 | Cost: 0.9319\n",
|
||||||
|
"Epoch: 008/020 | Batch 0200/0391 | Cost: 1.0566\n",
|
||||||
|
"Epoch: 008/020 | Batch 0250/0391 | Cost: 1.0591\n",
|
||||||
|
"Epoch: 008/020 | Batch 0300/0391 | Cost: 0.7914\n",
|
||||||
|
"Epoch: 008/020 | Batch 0350/0391 | Cost: 0.9090\n",
|
||||||
|
"Epoch: 008/020 | Train: 72.846% | Loss: 0.770\n",
|
||||||
|
"Time elapsed: 8.27 min\n",
|
||||||
|
"Epoch: 009/020 | Batch 0000/0391 | Cost: 0.6672\n",
|
||||||
|
"Epoch: 009/020 | Batch 0050/0391 | Cost: 0.7192\n",
|
||||||
|
"Epoch: 009/020 | Batch 0100/0391 | Cost: 0.8586\n",
|
||||||
|
"Epoch: 009/020 | Batch 0150/0391 | Cost: 0.7310\n",
|
||||||
|
"Epoch: 009/020 | Batch 0200/0391 | Cost: 0.8406\n",
|
||||||
|
"Epoch: 009/020 | Batch 0250/0391 | Cost: 0.7620\n",
|
||||||
|
"Epoch: 009/020 | Batch 0300/0391 | Cost: 0.6692\n",
|
||||||
|
"Epoch: 009/020 | Batch 0350/0391 | Cost: 0.6407\n",
|
||||||
|
"Epoch: 009/020 | Train: 73.702% | Loss: 0.748\n",
|
||||||
|
"Time elapsed: 9.30 min\n",
|
||||||
|
"Epoch: 010/020 | Batch 0000/0391 | Cost: 0.6539\n",
|
||||||
|
"Epoch: 010/020 | Batch 0050/0391 | Cost: 1.0382\n",
|
||||||
|
"Epoch: 010/020 | Batch 0100/0391 | Cost: 0.5921\n",
|
||||||
|
"Epoch: 010/020 | Batch 0150/0391 | Cost: 0.4933\n",
|
||||||
|
"Epoch: 010/020 | Batch 0200/0391 | Cost: 0.7485\n",
|
||||||
|
"Epoch: 010/020 | Batch 0250/0391 | Cost: 0.6779\n",
|
||||||
|
"Epoch: 010/020 | Batch 0300/0391 | Cost: 0.6787\n",
|
||||||
|
"Epoch: 010/020 | Batch 0350/0391 | Cost: 0.6977\n",
|
||||||
|
"Epoch: 010/020 | Train: 75.708% | Loss: 0.703\n",
|
||||||
|
"Time elapsed: 10.34 min\n",
|
||||||
|
"Epoch: 011/020 | Batch 0000/0391 | Cost: 0.6866\n",
|
||||||
|
"Epoch: 011/020 | Batch 0050/0391 | Cost: 0.7203\n",
|
||||||
|
"Epoch: 011/020 | Batch 0100/0391 | Cost: 0.5730\n",
|
||||||
|
"Epoch: 011/020 | Batch 0150/0391 | Cost: 0.5762\n",
|
||||||
|
"Epoch: 011/020 | Batch 0200/0391 | Cost: 0.6571\n",
|
||||||
|
"Epoch: 011/020 | Batch 0250/0391 | Cost: 0.7582\n",
|
||||||
|
"Epoch: 011/020 | Batch 0300/0391 | Cost: 0.7366\n",
|
||||||
|
"Epoch: 011/020 | Batch 0350/0391 | Cost: 0.6810\n",
|
||||||
|
"Epoch: 011/020 | Train: 79.044% | Loss: 0.606\n",
|
||||||
|
"Time elapsed: 11.37 min\n",
|
||||||
|
"Epoch: 012/020 | Batch 0000/0391 | Cost: 0.5665\n",
|
||||||
|
"Epoch: 012/020 | Batch 0050/0391 | Cost: 0.7081\n",
|
||||||
|
"Epoch: 012/020 | Batch 0100/0391 | Cost: 0.6823\n",
|
||||||
|
"Epoch: 012/020 | Batch 0150/0391 | Cost: 0.8297\n",
|
||||||
|
"Epoch: 012/020 | Batch 0200/0391 | Cost: 0.6470\n",
|
||||||
|
"Epoch: 012/020 | Batch 0250/0391 | Cost: 0.7293\n",
|
||||||
|
"Epoch: 012/020 | Batch 0300/0391 | Cost: 0.9127\n",
|
||||||
|
"Epoch: 012/020 | Batch 0350/0391 | Cost: 0.8419\n",
|
||||||
|
"Epoch: 012/020 | Train: 79.474% | Loss: 0.585\n",
|
||||||
|
"Time elapsed: 12.40 min\n",
|
||||||
|
"Epoch: 013/020 | Batch 0000/0391 | Cost: 0.4087\n",
|
||||||
|
"Epoch: 013/020 | Batch 0050/0391 | Cost: 0.4224\n",
|
||||||
|
"Epoch: 013/020 | Batch 0100/0391 | Cost: 0.4336\n",
|
||||||
|
"Epoch: 013/020 | Batch 0150/0391 | Cost: 0.6586\n",
|
||||||
|
"Epoch: 013/020 | Batch 0200/0391 | Cost: 0.7107\n",
|
||||||
|
"Epoch: 013/020 | Batch 0250/0391 | Cost: 0.7359\n",
|
||||||
|
"Epoch: 013/020 | Batch 0300/0391 | Cost: 0.4860\n",
|
||||||
|
"Epoch: 013/020 | Batch 0350/0391 | Cost: 0.7271\n",
|
||||||
|
"Epoch: 013/020 | Train: 80.746% | Loss: 0.549\n",
|
||||||
|
"Time elapsed: 13.44 min\n",
|
||||||
|
"Epoch: 014/020 | Batch 0000/0391 | Cost: 0.5500\n",
|
||||||
|
"Epoch: 014/020 | Batch 0050/0391 | Cost: 0.5108\n",
|
||||||
|
"Epoch: 014/020 | Batch 0100/0391 | Cost: 0.5186\n",
|
||||||
|
"Epoch: 014/020 | Batch 0150/0391 | Cost: 0.4737\n",
|
||||||
|
"Epoch: 014/020 | Batch 0200/0391 | Cost: 0.7015\n",
|
||||||
|
"Epoch: 014/020 | Batch 0250/0391 | Cost: 0.6069\n",
|
||||||
|
"Epoch: 014/020 | Batch 0300/0391 | Cost: 0.7080\n",
|
||||||
|
"Epoch: 014/020 | Batch 0350/0391 | Cost: 0.6460\n",
|
||||||
|
"Epoch: 014/020 | Train: 81.596% | Loss: 0.553\n",
|
||||||
|
"Time elapsed: 14.47 min\n",
|
||||||
|
"Epoch: 015/020 | Batch 0000/0391 | Cost: 0.5398\n",
|
||||||
|
"Epoch: 015/020 | Batch 0050/0391 | Cost: 0.5269\n",
|
||||||
|
"Epoch: 015/020 | Batch 0100/0391 | Cost: 0.5048\n",
|
||||||
|
"Epoch: 015/020 | Batch 0150/0391 | Cost: 0.5873\n",
|
||||||
|
"Epoch: 015/020 | Batch 0200/0391 | Cost: 0.5320\n",
|
||||||
|
"Epoch: 015/020 | Batch 0250/0391 | Cost: 0.4743\n",
|
||||||
|
"Epoch: 015/020 | Batch 0300/0391 | Cost: 0.6124\n",
|
||||||
|
"Epoch: 015/020 | Batch 0350/0391 | Cost: 0.7204\n",
|
||||||
|
"Epoch: 015/020 | Train: 85.276% | Loss: 0.439\n",
|
||||||
|
"Time elapsed: 15.51 min\n",
|
||||||
|
"Epoch: 016/020 | Batch 0000/0391 | Cost: 0.4387\n",
|
||||||
|
"Epoch: 016/020 | Batch 0050/0391 | Cost: 0.3777\n",
|
||||||
|
"Epoch: 016/020 | Batch 0100/0391 | Cost: 0.3430\n",
|
||||||
|
"Epoch: 016/020 | Batch 0150/0391 | Cost: 0.5901\n",
|
||||||
|
"Epoch: 016/020 | Batch 0200/0391 | Cost: 0.6303\n",
|
||||||
|
"Epoch: 016/020 | Batch 0250/0391 | Cost: 0.4983\n",
|
||||||
|
"Epoch: 016/020 | Batch 0300/0391 | Cost: 0.6507\n",
|
||||||
|
"Epoch: 016/020 | Batch 0350/0391 | Cost: 0.4663\n",
|
||||||
|
"Epoch: 016/020 | Train: 86.440% | Loss: 0.406\n",
|
||||||
|
"Time elapsed: 16.55 min\n",
|
||||||
|
"Epoch: 017/020 | Batch 0000/0391 | Cost: 0.4675\n",
|
||||||
|
"Epoch: 017/020 | Batch 0050/0391 | Cost: 0.6440\n",
|
||||||
|
"Epoch: 017/020 | Batch 0100/0391 | Cost: 0.3536\n",
|
||||||
|
"Epoch: 017/020 | Batch 0150/0391 | Cost: 0.5421\n",
|
||||||
|
"Epoch: 017/020 | Batch 0200/0391 | Cost: 0.4504\n",
|
||||||
|
"Epoch: 017/020 | Batch 0250/0391 | Cost: 0.4169\n",
|
||||||
|
"Epoch: 017/020 | Batch 0300/0391 | Cost: 0.4617\n",
|
||||||
|
"Epoch: 017/020 | Batch 0350/0391 | Cost: 0.4092\n",
|
||||||
|
"Epoch: 017/020 | Train: 84.636% | Loss: 0.459\n",
|
||||||
|
"Time elapsed: 17.59 min\n",
|
||||||
|
"Epoch: 018/020 | Batch 0000/0391 | Cost: 0.4267\n",
|
||||||
|
"Epoch: 018/020 | Batch 0050/0391 | Cost: 0.6478\n",
|
||||||
|
"Epoch: 018/020 | Batch 0100/0391 | Cost: 0.5806\n",
|
||||||
|
"Epoch: 018/020 | Batch 0150/0391 | Cost: 0.5453\n",
|
||||||
|
"Epoch: 018/020 | Batch 0200/0391 | Cost: 0.4984\n",
|
||||||
|
"Epoch: 018/020 | Batch 0250/0391 | Cost: 0.2517\n",
|
||||||
|
"Epoch: 018/020 | Batch 0300/0391 | Cost: 0.5219\n",
|
||||||
|
"Epoch: 018/020 | Batch 0350/0391 | Cost: 0.5217\n",
|
||||||
|
"Epoch: 018/020 | Train: 86.094% | Loss: 0.413\n",
|
||||||
|
"Time elapsed: 18.63 min\n",
|
||||||
|
"Epoch: 019/020 | Batch 0000/0391 | Cost: 0.3849\n",
|
||||||
|
"Epoch: 019/020 | Batch 0050/0391 | Cost: 0.2890\n",
|
||||||
|
"Epoch: 019/020 | Batch 0100/0391 | Cost: 0.5058\n",
|
||||||
|
"Epoch: 019/020 | Batch 0150/0391 | Cost: 0.5718\n",
|
||||||
|
"Epoch: 019/020 | Batch 0200/0391 | Cost: 0.4053\n",
|
||||||
|
"Epoch: 019/020 | Batch 0250/0391 | Cost: 0.5241\n",
|
||||||
|
"Epoch: 019/020 | Batch 0300/0391 | Cost: 0.7110\n",
|
||||||
|
"Epoch: 019/020 | Batch 0350/0391 | Cost: 0.4572\n",
|
||||||
|
"Epoch: 019/020 | Train: 87.586% | Loss: 0.365\n",
|
||||||
|
"Time elapsed: 19.67 min\n",
|
||||||
|
"Epoch: 020/020 | Batch 0000/0391 | Cost: 0.3576\n",
|
||||||
|
"Epoch: 020/020 | Batch 0050/0391 | Cost: 0.3466\n",
|
||||||
|
"Epoch: 020/020 | Batch 0100/0391 | Cost: 0.3427\n",
|
||||||
|
"Epoch: 020/020 | Batch 0150/0391 | Cost: 0.3117\n",
|
||||||
|
"Epoch: 020/020 | Batch 0200/0391 | Cost: 0.4912\n",
|
||||||
|
"Epoch: 020/020 | Batch 0250/0391 | Cost: 0.4481\n",
|
||||||
|
"Epoch: 020/020 | Batch 0300/0391 | Cost: 0.6303\n",
|
||||||
|
"Epoch: 020/020 | Batch 0350/0391 | Cost: 0.4274\n",
|
||||||
|
"Epoch: 020/020 | Train: 88.024% | Loss: 0.361\n",
|
||||||
|
"Time elapsed: 20.71 min\n",
|
||||||
|
"Total Training Time: 20.71 min\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def compute_accuracy(model, data_loader):\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" correct_pred, num_examples = 0, 0\n",
|
||||||
|
" for i, (features, targets) in enumerate(data_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
"\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" _, predicted_labels = torch.max(probas, 1)\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" correct_pred += (predicted_labels == targets).sum()\n",
|
||||||
|
" return correct_pred.float()/num_examples * 100\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def compute_epoch_loss(model, data_loader):\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" curr_loss, num_examples = 0., 0\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" for features, targets in data_loader:\n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" loss = F.cross_entropy(logits, targets, reduction='sum')\n",
|
||||||
|
" num_examples += targets.size(0)\n",
|
||||||
|
" curr_loss += loss\n",
|
||||||
|
"\n",
|
||||||
|
" curr_loss = curr_loss / num_examples\n",
|
||||||
|
" return curr_loss\n",
|
||||||
|
" \n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"start_time = time.time()\n",
|
||||||
|
"for epoch in range(num_epochs):\n",
|
||||||
|
" \n",
|
||||||
|
" model.train()\n",
|
||||||
|
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
|
||||||
|
" \n",
|
||||||
|
" features = features.to(DEVICE)\n",
|
||||||
|
" targets = targets.to(DEVICE)\n",
|
||||||
|
" \n",
|
||||||
|
" ### FORWARD AND BACK PROP\n",
|
||||||
|
" logits, probas = model(features)\n",
|
||||||
|
" cost = F.cross_entropy(logits, targets)\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" cost.backward()\n",
|
||||||
|
" \n",
|
||||||
|
" ### UPDATE MODEL PARAMETERS\n",
|
||||||
|
" optimizer.step()\n",
|
||||||
|
" \n",
|
||||||
|
" ### LOGGING\n",
|
||||||
|
" if not batch_idx % 50:\n",
|
||||||
|
" print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
|
||||||
|
" %(epoch+1, num_epochs, batch_idx, \n",
|
||||||
|
" len(train_loader), cost))\n",
|
||||||
|
"\n",
|
||||||
|
" model.eval()\n",
|
||||||
|
" with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Epoch: %03d/%03d | Train: %.3f%% | Loss: %.3f' % (\n",
|
||||||
|
" epoch+1, num_epochs, \n",
|
||||||
|
" compute_accuracy(model, train_loader),\n",
|
||||||
|
" compute_epoch_loss(model, train_loader)))\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
|
||||||
|
" \n",
|
||||||
|
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "paaeEQHQj5xC"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Evaluation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"autoexec": {
|
||||||
|
"startup": false,
|
||||||
|
"wait_interval": 0
|
||||||
|
},
|
||||||
|
"base_uri": "https://localhost:8080/",
|
||||||
|
"height": 34
|
||||||
|
},
|
||||||
|
"colab_type": "code",
|
||||||
|
"executionInfo": {
|
||||||
|
"elapsed": 6514,
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": 1524976895054,
|
||||||
|
"user": {
|
||||||
|
"displayName": "Sebastian Raschka",
|
||||||
|
"photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
|
||||||
|
"userId": "118404394130788869227"
|
||||||
|
},
|
||||||
|
"user_tz": 240
|
||||||
|
},
|
||||||
|
"id": "gzQMWKq5j5xE",
|
||||||
|
"outputId": "de7dc005-5eeb-4177-9f9f-d9b5d1358db9"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test accuracy: 74.56%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"with torch.set_grad_enabled(False): # save memory during inference\n",
|
||||||
|
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"numpy 1.15.4\n",
|
||||||
|
"torch 1.0.1.post2\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"%watermark -iv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"accelerator": "GPU",
|
||||||
|
"colab": {
|
||||||
|
"collapsed_sections": [],
|
||||||
|
"default_view": {},
|
||||||
|
"name": "convnet-vgg16.ipynb",
|
||||||
|
"provenance": [],
|
||||||
|
"version": "0.3.2",
|
||||||
|
"views": {}
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.1"
|
||||||
|
},
|
||||||
|
"toc": {
|
||||||
|
"nav_menu": {},
|
||||||
|
"number_sections": true,
|
||||||
|
"sideBar": true,
|
||||||
|
"skip_h1_title": false,
|
||||||
|
"title_cell": "Table of Contents",
|
||||||
|
"title_sidebar": "Contents",
|
||||||
|
"toc_cell": true,
|
||||||
|
"toc_position": {
|
||||||
|
"height": "calc(100% - 180px)",
|
||||||
|
"left": "10px",
|
||||||
|
"top": "150px",
|
||||||
|
"width": "371px"
|
||||||
|
},
|
||||||
|
"toc_section_display": true,
|
||||||
|
"toc_window_display": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
@@ -0,0 +1,263 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
|
||||||
|
"- Author: Sebastian Raschka\n",
|
||||||
|
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%load_ext watermark\n",
|
||||||
|
"%watermark -a 'Sebastian Raschka' -v -p torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Replacing Fully-Connnected by Equivalent Convolutional Layers"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 15,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import torch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Assume we have a 2x2 input image:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 16,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"torch.Size([1, 1, 2, 2])"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 16,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"inputs = torch.tensor([[[[1., 2.],\n",
|
||||||
|
" [3., 4.]]]])\n",
|
||||||
|
"\n",
|
||||||
|
"inputs.shape"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Fully Connected"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"A fully connected layer, which maps the 4 input features two 2 outputs, would be computed as follows:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 17,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"fc = torch.nn.Linear(4, 2)\n",
|
||||||
|
"\n",
|
||||||
|
"weights = torch.tensor([[1.1, 1.2, 1.3, 1.4],\n",
|
||||||
|
" [1.5, 1.6, 1.7, 1.8]])\n",
|
||||||
|
"bias = torch.tensor([1.9, 2.0])\n",
|
||||||
|
"fc.weight.data = weights\n",
|
||||||
|
"fc.bias.data = bias"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 18,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"tensor([[14.9000, 19.0000]], grad_fn=<ReluBackward0>)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 18,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"torch.relu(fc(inputs.view(-1, 4)))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Convolution with Kernels equal to the input size"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"We can obtain the same outputs if we use convolutional layers where the kernel size is the same size as the input feature array:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 19,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"torch.Size([2, 1, 2, 2])\n",
|
||||||
|
"torch.Size([2])\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"conv = torch.nn.Conv2d(in_channels=1,\n",
|
||||||
|
" out_channels=2,\n",
|
||||||
|
" kernel_size=inputs.squeeze(dim=(0)).squeeze(dim=(0)).size())\n",
|
||||||
|
"print(conv.weight.size())\n",
|
||||||
|
"print(conv.bias.size())"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 20,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"conv.weight.data = weights.view(2, 1, 2, 2)\n",
|
||||||
|
"conv.bias.data = bias"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 21,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"tensor([[[[14.9000]],\n",
|
||||||
|
"\n",
|
||||||
|
" [[19.0000]]]], grad_fn=<ReluBackward0>)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 21,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"torch.relu(conv(inputs))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Convolution with 1x1 Kernels"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Similarly, we can replace the fully connected layer using a convolutional layer when we reshape the input image into a num_inputs x 1 x 1 image:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 23,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"tensor([[[[14.9000]],\n",
|
||||||
|
"\n",
|
||||||
|
" [[19.0000]]]], grad_fn=<ReluBackward0>)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 23,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"conv = torch.nn.Conv2d(in_channels=4,\n",
|
||||||
|
" out_channels=2,\n",
|
||||||
|
" kernel_size=(1, 1))\n",
|
||||||
|
"\n",
|
||||||
|
"conv.weight.data = weights.view(2, 4, 1, 1)\n",
|
||||||
|
"conv.bias.data = bias\n",
|
||||||
|
"torch.relu(conv(inputs.view(1, 4, 1, 1)))"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user