wehub/fchollet--deep-learning-with-python-notebooks
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Packages Projects Releases Wiki Activity
Files
main
fchollet--deep-learning-wit…/chapter05_fundamentals-of-ml.ipynb
T
wehub-resource-sync 612b28061b chore: import upstream snapshot with attribution
2026-07-13 13:25:23 +08:00

985 lines
23 KiB
Plaintext
Raw Permalink Blame History

{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"This is a companion notebook for the book [Deep Learning with Python, Third Edition](https://www.manning.com/books/deep-learning-with-python-third-edition). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n\n**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n\nThe book's contents are available online at [deeplearningwithpython.io](https://deeplearningwithpython.io)."
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"!pip install keras keras-hub --upgrade -q"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"KERAS_BACKEND\"] = \"jax\""
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"cellView": "form",
"colab_type": "code"
},
"outputs": [],
"source": [
"# @title\n",
"import os\n",
"from IPython.core.magic import register_cell_magic\n",
"\n",
"@register_cell_magic\n",
"def backend(line, cell):\n",
" current, required = os.environ.get(\"KERAS_BACKEND\", \"\"), line.split()[-1]\n",
" if current == required:\n",
" get_ipython().run_cell(cell)\n",
" else:\n",
" print(\n",
" f\"This cell requires the {required} backend. To run it, change KERAS_BACKEND to \"\n",
" f\"\\\"{required}\\\" at the top of the notebook, restart the runtime, and rerun the notebook.\"\n",
" )"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"## Fundamentals of machine learning"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"### Generalization: The goal of machine learning"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Underfitting and overfitting"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Noisy training data"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Ambiguous features"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Rare features and spurious correlations"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"from keras.datasets import mnist\n",
"import numpy as np\n",
"\n",
"(train_images, train_labels), _ = mnist.load_data()\n",
"train_images = train_images.reshape((60000, 28 * 28))\n",
"train_images = train_images.astype(\"float32\") / 255\n",
"\n",
"train_images_with_noise_channels = np.concatenate(\n",
" [train_images, np.random.random((len(train_images), 784))], axis=1\n",
")\n",
"\n",
"train_images_with_zeros_channels = np.concatenate(\n",
" [train_images, np.zeros((len(train_images), 784))], axis=1\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"import keras\n",
"from keras import layers\n",
"\n",
"def get_model():\n",
" model = keras.Sequential(\n",
" [\n",
" layers.Dense(512, activation=\"relu\"),\n",
" layers.Dense(10, activation=\"softmax\"),\n",
" ]\n",
" )\n",
" model.compile(\n",
" optimizer=\"adam\",\n",
" loss=\"sparse_categorical_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
" )\n",
" return model\n",
"\n",
"model = get_model()\n",
"history_noise = model.fit(\n",
" train_images_with_noise_channels,\n",
" train_labels,\n",
" epochs=10,\n",
" batch_size=128,\n",
" validation_split=0.2,\n",
")\n",
"\n",
"model = get_model()\n",
"history_zeros = model.fit(\n",
" train_images_with_zeros_channels,\n",
" train_labels,\n",
" epochs=10,\n",
" batch_size=128,\n",
" validation_split=0.2,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"\n",
"val_acc_noise = history_noise.history[\"val_accuracy\"]\n",
"val_acc_zeros = history_zeros.history[\"val_accuracy\"]\n",
"epochs = range(1, 11)\n",
"plt.plot(\n",
" epochs,\n",
" val_acc_noise,\n",
" \"b-\",\n",
" label=\"Validation accuracy with noise channels\",\n",
")\n",
"plt.plot(\n",
" epochs,\n",
" val_acc_zeros,\n",
" \"r--\",\n",
" label=\"Validation accuracy with zeros channels\",\n",
")\n",
"plt.title(\"Effect of noise channels on validation accuracy\")\n",
"plt.xlabel(\"Epochs\")\n",
"plt.xticks(epochs)\n",
"plt.ylabel(\"Accuracy\")\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### The nature of generalization in deep learning"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"(train_images, train_labels), _ = mnist.load_data()\n",
"train_images = train_images.reshape((60000, 28 * 28))\n",
"train_images = train_images.astype(\"float32\") / 255\n",
"\n",
"random_train_labels = train_labels[:]\n",
"np.random.shuffle(random_train_labels)\n",
"\n",
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(512, activation=\"relu\"),\n",
" layers.Dense(10, activation=\"softmax\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=\"rmsprop\",\n",
" loss=\"sparse_categorical_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"model.fit(\n",
" train_images,\n",
" random_train_labels,\n",
" epochs=100,\n",
" batch_size=128,\n",
" validation_split=0.2,\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### The manifold hypothesis"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Interpolation as a source of generalization"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Why deep learning works"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Training data is paramount"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"### Evaluating machine-learning models"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Training, validation, and test sets"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Simple hold-out validation"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### K-fold validation"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Iterated K-fold validation with shuffling"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Beating a common-sense baseline"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Things to keep in mind about model evaluation"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"### Improving model fit"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Tuning key gradient descent parameters"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"(train_images, train_labels), _ = mnist.load_data()\n",
"train_images = train_images.reshape((60000, 28 * 28))\n",
"train_images = train_images.astype(\"float32\") / 255\n",
"\n",
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(512, activation=\"relu\"),\n",
" layers.Dense(10, activation=\"softmax\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=keras.optimizers.RMSprop(learning_rate=1.0),\n",
" loss=\"sparse_categorical_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"model.fit(\n",
" train_images, train_labels, epochs=10, batch_size=128, validation_split=0.2\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(512, activation=\"relu\"),\n",
" layers.Dense(10, activation=\"softmax\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=keras.optimizers.RMSprop(learning_rate=1e-2),\n",
" loss=\"sparse_categorical_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"model.fit(\n",
" train_images, train_labels, epochs=10, batch_size=128, validation_split=0.2\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Using better architecture priors"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Increasing model capacity"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"model = keras.Sequential([layers.Dense(10, activation=\"softmax\")])\n",
"model.compile(\n",
" optimizer=\"rmsprop\",\n",
" loss=\"sparse_categorical_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"history_small_model = model.fit(\n",
" train_images, train_labels, epochs=20, batch_size=128, validation_split=0.2\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"\n",
"val_loss = history_small_model.history[\"val_loss\"]\n",
"epochs = range(1, 21)\n",
"plt.plot(epochs, val_loss, \"b-\", label=\"Validation loss\")\n",
"plt.title(\"Validation loss for a model with insufficient capacity\")\n",
"plt.xlabel(\"Epochs\")\n",
"plt.ylabel(\"Loss\")\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(128, activation=\"relu\"),\n",
" layers.Dense(128, activation=\"relu\"),\n",
" layers.Dense(10, activation=\"softmax\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=\"rmsprop\",\n",
" loss=\"sparse_categorical_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"history_large_model = model.fit(\n",
" train_images,\n",
" train_labels,\n",
" epochs=20,\n",
" batch_size=128,\n",
" validation_split=0.2,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"val_loss = history_large_model.history[\"val_loss\"]\n",
"epochs = range(1, 21)\n",
"plt.plot(epochs, val_loss, \"b-\", label=\"Validation loss\")\n",
"plt.title(\"Validation loss for a model with appropriate capacity\")\n",
"plt.xlabel(\"Epochs\")\n",
"plt.ylabel(\"Loss\")\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(2048, activation=\"relu\"),\n",
" layers.Dense(2048, activation=\"relu\"),\n",
" layers.Dense(2048, activation=\"relu\"),\n",
" layers.Dense(10, activation=\"softmax\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=\"rmsprop\",\n",
" loss=\"sparse_categorical_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"history_very_large_model = model.fit(\n",
" train_images,\n",
" train_labels,\n",
" epochs=20,\n",
" batch_size=32,\n",
" validation_split=0.2,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"val_loss = history_very_large_model.history[\"val_loss\"]\n",
"epochs = range(1, 21)\n",
"plt.plot(epochs, val_loss, \"b-\", label=\"Validation loss\")\n",
"plt.title(\"Validation loss for a model with too much capacity\")\n",
"plt.xlabel(\"Epochs\")\n",
"plt.ylabel(\"Loss\")\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"### Improving generalization"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Dataset curation"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Feature engineering"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Using early stopping"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"#### Regularizing your model"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Reducing the network's size"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"from keras.datasets import imdb\n",
"\n",
"(train_data, train_labels), _ = imdb.load_data(num_words=10000)\n",
"\n",
"def vectorize_sequences(sequences, dimension=10000):\n",
" results = np.zeros((len(sequences), dimension))\n",
" for i, sequence in enumerate(sequences):\n",
" results[i, sequence] = 1.0\n",
" return results\n",
"\n",
"train_data = vectorize_sequences(train_data)\n",
"\n",
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(16, activation=\"relu\"),\n",
" layers.Dense(16, activation=\"relu\"),\n",
" layers.Dense(1, activation=\"sigmoid\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=\"rmsprop\",\n",
" loss=\"binary_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"history_original = model.fit(\n",
" train_data,\n",
" train_labels,\n",
" epochs=20,\n",
" batch_size=512,\n",
" validation_split=0.4,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(4, activation=\"relu\"),\n",
" layers.Dense(4, activation=\"relu\"),\n",
" layers.Dense(1, activation=\"sigmoid\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=\"rmsprop\",\n",
" loss=\"binary_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"history_smaller_model = model.fit(\n",
" train_data,\n",
" train_labels,\n",
" epochs=20,\n",
" batch_size=512,\n",
" validation_split=0.4,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"original_val_loss = history_original.history[\"val_loss\"]\n",
"smaller_model_val_loss = history_smaller_model.history[\"val_loss\"]\n",
"epochs = range(1, 21)\n",
"plt.plot(\n",
" epochs,\n",
" original_val_loss,\n",
" \"r--\",\n",
" label=\"Validation loss of original model\",\n",
")\n",
"plt.plot(\n",
" epochs,\n",
" smaller_model_val_loss,\n",
" \"b-\",\n",
" label=\"Validation loss of smaller model\",\n",
")\n",
"plt.title(\"Original model vs. smaller model (IMDB review classification)\")\n",
"plt.xlabel(\"Epochs\")\n",
"plt.ylabel(\"Loss\")\n",
"plt.xticks(epochs)\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(512, activation=\"relu\"),\n",
" layers.Dense(512, activation=\"relu\"),\n",
" layers.Dense(1, activation=\"sigmoid\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=\"rmsprop\",\n",
" loss=\"binary_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"history_larger_model = model.fit(\n",
" train_data,\n",
" train_labels,\n",
" epochs=20,\n",
" batch_size=512,\n",
" validation_split=0.4,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"original_val_loss = history_original.history[\"val_loss\"]\n",
"larger_model_val_loss = history_larger_model.history[\"val_loss\"]\n",
"epochs = range(1, 21)\n",
"plt.plot(\n",
" epochs,\n",
" original_val_loss,\n",
" \"r--\",\n",
" label=\"Validation loss of original model\",\n",
")\n",
"plt.plot(\n",
" epochs,\n",
" larger_model_val_loss,\n",
" \"b-\",\n",
" label=\"Validation loss of larger model\",\n",
")\n",
"plt.title(\"Original model vs. larger model (IMDB review classification)\")\n",
"plt.xlabel(\"Epochs\")\n",
"plt.ylabel(\"Loss\")\n",
"plt.xticks(epochs)\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Adding weight regularization"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"from keras.regularizers import l2\n",
"\n",
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(16, kernel_regularizer=l2(0.002), activation=\"relu\"),\n",
" layers.Dense(16, kernel_regularizer=l2(0.002), activation=\"relu\"),\n",
" layers.Dense(1, activation=\"sigmoid\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=\"rmsprop\",\n",
" loss=\"binary_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"history_l2_reg = model.fit(\n",
" train_data,\n",
" train_labels,\n",
" epochs=20,\n",
" batch_size=512,\n",
" validation_split=0.4,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"original_val_loss = history_original.history[\"val_loss\"]\n",
"l2_val_loss = history_l2_reg.history[\"val_loss\"]\n",
"epochs = range(1, 21)\n",
"plt.plot(\n",
" epochs,\n",
" original_val_loss,\n",
" \"r--\",\n",
" label=\"Validation loss of original model\",\n",
")\n",
"plt.plot(\n",
" epochs,\n",
" l2_val_loss,\n",
" \"b-\",\n",
" label=\"Validation loss of L2-regularized model\",\n",
")\n",
"plt.title(\n",
" \"Original model vs. L2-regularized model (IMDB review classification)\"\n",
")\n",
"plt.xlabel(\"Epochs\")\n",
"plt.ylabel(\"Loss\")\n",
"plt.xticks(epochs)\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"from keras import regularizers\n",
"\n",
"regularizers.l1(0.001)\n",
"regularizers.l1_l2(l1=0.001, l2=0.001)"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text"
},
"source": [
"##### Adding dropout"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"model = keras.Sequential(\n",
" [\n",
" layers.Dense(16, activation=\"relu\"),\n",
" layers.Dropout(0.5),\n",
" layers.Dense(16, activation=\"relu\"),\n",
" layers.Dropout(0.5),\n",
" layers.Dense(1, activation=\"sigmoid\"),\n",
" ]\n",
")\n",
"model.compile(\n",
" optimizer=\"rmsprop\",\n",
" loss=\"binary_crossentropy\",\n",
" metrics=[\"accuracy\"],\n",
")\n",
"history_dropout = model.fit(\n",
" train_data,\n",
" train_labels,\n",
" epochs=20,\n",
" batch_size=512,\n",
" validation_split=0.4,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"original_val_loss = history_original.history[\"val_loss\"]\n",
"dropout_val_loss = history_dropout.history[\"val_loss\"]\n",
"epochs = range(1, 21)\n",
"plt.plot(\n",
" epochs,\n",
" original_val_loss,\n",
" \"r--\",\n",
" label=\"Validation loss of original model\",\n",
")\n",
"plt.plot(\n",
" epochs,\n",
" dropout_val_loss,\n",
" \"b-\",\n",
" label=\"Validation loss of dropout-regularized model\",\n",
")\n",
"plt.title(\n",
" \"Original model vs. dropout-regularized model (IMDB review classification)\"\n",
")\n",
"plt.xlabel(\"Epochs\")\n",
"plt.ylabel(\"Loss\")\n",
"plt.xticks(epochs)\n",
"plt.legend()\n",
"plt.show()"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"collapsed_sections": [],
"name": "chapter05_fundamentals-of-ml",
"private_outputs": false,
"provenance": [],
"toc_visible": true
},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.0"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Reference in New Issue View Git Blame Copy Permalink