{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "This is a companion notebook for the book [Deep Learning with Python, Third Edition](https://www.manning.com/books/deep-learning-with-python-third-edition). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n\n**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n\nThe book's contents are available online at [deeplearningwithpython.io](https://deeplearningwithpython.io)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!pip install keras keras-hub --upgrade -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"jax\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "cellView": "form",
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# @title\n",
    "import os\n",
    "from IPython.core.magic import register_cell_magic\n",
    "\n",
    "@register_cell_magic\n",
    "def backend(line, cell):\n",
    "    current, required = os.environ.get(\"KERAS_BACKEND\", \"\"), line.split()[-1]\n",
    "    if current == required:\n",
    "        get_ipython().run_cell(cell)\n",
    "    else:\n",
    "        print(\n",
    "            f\"This cell requires the {required} backend. To run it, change KERAS_BACKEND to \"\n",
    "            f\"\\\"{required}\\\" at the top of the notebook, restart the runtime, and rerun the notebook.\"\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Fundamentals of machine learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Generalization: The goal of machine learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Underfitting and overfitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Noisy training data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Ambiguous features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Rare features and spurious correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from keras.datasets import mnist\n",
    "import numpy as np\n",
    "\n",
    "(train_images, train_labels), _ = mnist.load_data()\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "\n",
    "train_images_with_noise_channels = np.concatenate(\n",
    "    [train_images, np.random.random((len(train_images), 784))], axis=1\n",
    ")\n",
    "\n",
    "train_images_with_zeros_channels = np.concatenate(\n",
    "    [train_images, np.zeros((len(train_images), 784))], axis=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras import layers\n",
    "\n",
    "def get_model():\n",
    "    model = keras.Sequential(\n",
    "        [\n",
    "            layers.Dense(512, activation=\"relu\"),\n",
    "            layers.Dense(10, activation=\"softmax\"),\n",
    "        ]\n",
    "    )\n",
    "    model.compile(\n",
    "        optimizer=\"adam\",\n",
    "        loss=\"sparse_categorical_crossentropy\",\n",
    "        metrics=[\"accuracy\"],\n",
    "    )\n",
    "    return model\n",
    "\n",
    "model = get_model()\n",
    "history_noise = model.fit(\n",
    "    train_images_with_noise_channels,\n",
    "    train_labels,\n",
    "    epochs=10,\n",
    "    batch_size=128,\n",
    "    validation_split=0.2,\n",
    ")\n",
    "\n",
    "model = get_model()\n",
    "history_zeros = model.fit(\n",
    "    train_images_with_zeros_channels,\n",
    "    train_labels,\n",
    "    epochs=10,\n",
    "    batch_size=128,\n",
    "    validation_split=0.2,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "val_acc_noise = history_noise.history[\"val_accuracy\"]\n",
    "val_acc_zeros = history_zeros.history[\"val_accuracy\"]\n",
    "epochs = range(1, 11)\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    val_acc_noise,\n",
    "    \"b-\",\n",
    "    label=\"Validation accuracy with noise channels\",\n",
    ")\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    val_acc_zeros,\n",
    "    \"r--\",\n",
    "    label=\"Validation accuracy with zeros channels\",\n",
    ")\n",
    "plt.title(\"Effect of noise channels on validation accuracy\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.xticks(epochs)\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### The nature of generalization in deep learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "(train_images, train_labels), _ = mnist.load_data()\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "\n",
    "random_train_labels = train_labels[:]\n",
    "np.random.shuffle(random_train_labels)\n",
    "\n",
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(512, activation=\"relu\"),\n",
    "        layers.Dense(10, activation=\"softmax\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=\"rmsprop\",\n",
    "    loss=\"sparse_categorical_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "model.fit(\n",
    "    train_images,\n",
    "    random_train_labels,\n",
    "    epochs=100,\n",
    "    batch_size=128,\n",
    "    validation_split=0.2,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### The manifold hypothesis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Interpolation as a source of generalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Why deep learning works"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Training data is paramount"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Evaluating machine-learning models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Training, validation, and test sets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Simple hold-out validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### K-fold validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Iterated K-fold validation with shuffling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Beating a common-sense baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Things to keep in mind about model evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Improving model fit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Tuning key gradient descent parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "(train_images, train_labels), _ = mnist.load_data()\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "\n",
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(512, activation=\"relu\"),\n",
    "        layers.Dense(10, activation=\"softmax\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=keras.optimizers.RMSprop(learning_rate=1.0),\n",
    "    loss=\"sparse_categorical_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "model.fit(\n",
    "    train_images, train_labels, epochs=10, batch_size=128, validation_split=0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(512, activation=\"relu\"),\n",
    "        layers.Dense(10, activation=\"softmax\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=keras.optimizers.RMSprop(learning_rate=1e-2),\n",
    "    loss=\"sparse_categorical_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "model.fit(\n",
    "    train_images, train_labels, epochs=10, batch_size=128, validation_split=0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Using better architecture priors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Increasing model capacity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential([layers.Dense(10, activation=\"softmax\")])\n",
    "model.compile(\n",
    "    optimizer=\"rmsprop\",\n",
    "    loss=\"sparse_categorical_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "history_small_model = model.fit(\n",
    "    train_images, train_labels, epochs=20, batch_size=128, validation_split=0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "val_loss = history_small_model.history[\"val_loss\"]\n",
    "epochs = range(1, 21)\n",
    "plt.plot(epochs, val_loss, \"b-\", label=\"Validation loss\")\n",
    "plt.title(\"Validation loss for a model with insufficient capacity\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(128, activation=\"relu\"),\n",
    "        layers.Dense(128, activation=\"relu\"),\n",
    "        layers.Dense(10, activation=\"softmax\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=\"rmsprop\",\n",
    "    loss=\"sparse_categorical_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "history_large_model = model.fit(\n",
    "    train_images,\n",
    "    train_labels,\n",
    "    epochs=20,\n",
    "    batch_size=128,\n",
    "    validation_split=0.2,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "val_loss = history_large_model.history[\"val_loss\"]\n",
    "epochs = range(1, 21)\n",
    "plt.plot(epochs, val_loss, \"b-\", label=\"Validation loss\")\n",
    "plt.title(\"Validation loss for a model with appropriate capacity\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(2048, activation=\"relu\"),\n",
    "        layers.Dense(2048, activation=\"relu\"),\n",
    "        layers.Dense(2048, activation=\"relu\"),\n",
    "        layers.Dense(10, activation=\"softmax\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=\"rmsprop\",\n",
    "    loss=\"sparse_categorical_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "history_very_large_model = model.fit(\n",
    "    train_images,\n",
    "    train_labels,\n",
    "    epochs=20,\n",
    "    batch_size=32,\n",
    "    validation_split=0.2,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "val_loss = history_very_large_model.history[\"val_loss\"]\n",
    "epochs = range(1, 21)\n",
    "plt.plot(epochs, val_loss, \"b-\", label=\"Validation loss\")\n",
    "plt.title(\"Validation loss for a model with too much capacity\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Improving generalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Dataset curation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Feature engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Using early stopping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Regularizing your model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Reducing the network's size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from keras.datasets import imdb\n",
    "\n",
    "(train_data, train_labels), _ = imdb.load_data(num_words=10000)\n",
    "\n",
    "def vectorize_sequences(sequences, dimension=10000):\n",
    "    results = np.zeros((len(sequences), dimension))\n",
    "    for i, sequence in enumerate(sequences):\n",
    "        results[i, sequence] = 1.0\n",
    "    return results\n",
    "\n",
    "train_data = vectorize_sequences(train_data)\n",
    "\n",
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(16, activation=\"relu\"),\n",
    "        layers.Dense(16, activation=\"relu\"),\n",
    "        layers.Dense(1, activation=\"sigmoid\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=\"rmsprop\",\n",
    "    loss=\"binary_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "history_original = model.fit(\n",
    "    train_data,\n",
    "    train_labels,\n",
    "    epochs=20,\n",
    "    batch_size=512,\n",
    "    validation_split=0.4,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(4, activation=\"relu\"),\n",
    "        layers.Dense(4, activation=\"relu\"),\n",
    "        layers.Dense(1, activation=\"sigmoid\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=\"rmsprop\",\n",
    "    loss=\"binary_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "history_smaller_model = model.fit(\n",
    "    train_data,\n",
    "    train_labels,\n",
    "    epochs=20,\n",
    "    batch_size=512,\n",
    "    validation_split=0.4,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "original_val_loss = history_original.history[\"val_loss\"]\n",
    "smaller_model_val_loss = history_smaller_model.history[\"val_loss\"]\n",
    "epochs = range(1, 21)\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    original_val_loss,\n",
    "    \"r--\",\n",
    "    label=\"Validation loss of original model\",\n",
    ")\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    smaller_model_val_loss,\n",
    "    \"b-\",\n",
    "    label=\"Validation loss of smaller model\",\n",
    ")\n",
    "plt.title(\"Original model vs. smaller model (IMDB review classification)\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.xticks(epochs)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(512, activation=\"relu\"),\n",
    "        layers.Dense(512, activation=\"relu\"),\n",
    "        layers.Dense(1, activation=\"sigmoid\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=\"rmsprop\",\n",
    "    loss=\"binary_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "history_larger_model = model.fit(\n",
    "    train_data,\n",
    "    train_labels,\n",
    "    epochs=20,\n",
    "    batch_size=512,\n",
    "    validation_split=0.4,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "original_val_loss = history_original.history[\"val_loss\"]\n",
    "larger_model_val_loss = history_larger_model.history[\"val_loss\"]\n",
    "epochs = range(1, 21)\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    original_val_loss,\n",
    "    \"r--\",\n",
    "    label=\"Validation loss of original model\",\n",
    ")\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    larger_model_val_loss,\n",
    "    \"b-\",\n",
    "    label=\"Validation loss of larger model\",\n",
    ")\n",
    "plt.title(\"Original model vs. larger model (IMDB review classification)\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.xticks(epochs)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Adding weight regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from keras.regularizers import l2\n",
    "\n",
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(16, kernel_regularizer=l2(0.002), activation=\"relu\"),\n",
    "        layers.Dense(16, kernel_regularizer=l2(0.002), activation=\"relu\"),\n",
    "        layers.Dense(1, activation=\"sigmoid\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=\"rmsprop\",\n",
    "    loss=\"binary_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "history_l2_reg = model.fit(\n",
    "    train_data,\n",
    "    train_labels,\n",
    "    epochs=20,\n",
    "    batch_size=512,\n",
    "    validation_split=0.4,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "original_val_loss = history_original.history[\"val_loss\"]\n",
    "l2_val_loss = history_l2_reg.history[\"val_loss\"]\n",
    "epochs = range(1, 21)\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    original_val_loss,\n",
    "    \"r--\",\n",
    "    label=\"Validation loss of original model\",\n",
    ")\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    l2_val_loss,\n",
    "    \"b-\",\n",
    "    label=\"Validation loss of L2-regularized model\",\n",
    ")\n",
    "plt.title(\n",
    "    \"Original model vs. L2-regularized model (IMDB review classification)\"\n",
    ")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.xticks(epochs)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from keras import regularizers\n",
    "\n",
    "regularizers.l1(0.001)\n",
    "regularizers.l1_l2(l1=0.001, l2=0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "##### Adding dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential(\n",
    "    [\n",
    "        layers.Dense(16, activation=\"relu\"),\n",
    "        layers.Dropout(0.5),\n",
    "        layers.Dense(16, activation=\"relu\"),\n",
    "        layers.Dropout(0.5),\n",
    "        layers.Dense(1, activation=\"sigmoid\"),\n",
    "    ]\n",
    ")\n",
    "model.compile(\n",
    "    optimizer=\"rmsprop\",\n",
    "    loss=\"binary_crossentropy\",\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "history_dropout = model.fit(\n",
    "    train_data,\n",
    "    train_labels,\n",
    "    epochs=20,\n",
    "    batch_size=512,\n",
    "    validation_split=0.4,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "original_val_loss = history_original.history[\"val_loss\"]\n",
    "dropout_val_loss = history_dropout.history[\"val_loss\"]\n",
    "epochs = range(1, 21)\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    original_val_loss,\n",
    "    \"r--\",\n",
    "    label=\"Validation loss of original model\",\n",
    ")\n",
    "plt.plot(\n",
    "    epochs,\n",
    "    dropout_val_loss,\n",
    "    \"b-\",\n",
    "    label=\"Validation loss of dropout-regularized model\",\n",
    ")\n",
    "plt.title(\n",
    "    \"Original model vs. dropout-regularized model (IMDB review classification)\"\n",
    ")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.xticks(epochs)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "chapter05_fundamentals-of-ml",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}