{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "This is a companion notebook for the book [Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition?a_aid=keras&a_bid=76564dff). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n\n**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n\nThis notebook was generated for TensorFlow 2.6."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "# Best practices for the real world"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Getting the most out of your models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Hyperparameter optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Using KerasTuner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!pip install keras-tuner -q"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**A KerasTuner model-building function**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "def build_model(hp):\n",
    "    units = hp.Int(name=\"units\", min_value=16, max_value=64, step=16)\n",
    "    model = keras.Sequential([\n",
    "        layers.Dense(units, activation=\"relu\"),\n",
    "        layers.Dense(10, activation=\"softmax\")\n",
    "    ])\n",
    "    optimizer = hp.Choice(name=\"optimizer\", values=[\"rmsprop\", \"adam\"])\n",
    "    model.compile(\n",
    "        optimizer=optimizer,\n",
    "        loss=\"sparse_categorical_crossentropy\",\n",
    "        metrics=[\"accuracy\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**A KerasTuner `HyperModel`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import kerastuner as kt\n",
    "\n",
    "class SimpleMLP(kt.HyperModel):\n",
    "    def __init__(self, num_classes):\n",
    "        self.num_classes = num_classes\n",
    "\n",
    "    def build(self, hp):\n",
    "        units = hp.Int(name=\"units\", min_value=16, max_value=64, step=16)\n",
    "        model = keras.Sequential([\n",
    "            layers.Dense(units, activation=\"relu\"),\n",
    "            layers.Dense(self.num_classes, activation=\"softmax\")\n",
    "        ])\n",
    "        optimizer = hp.Choice(name=\"optimizer\", values=[\"rmsprop\", \"adam\"])\n",
    "        model.compile(\n",
    "            optimizer=optimizer,\n",
    "            loss=\"sparse_categorical_crossentropy\",\n",
    "            metrics=[\"accuracy\"])\n",
    "        return model\n",
    "\n",
    "hypermodel = SimpleMLP(num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "tuner = kt.BayesianOptimization(\n",
    "    build_model,\n",
    "    objective=\"val_accuracy\",\n",
    "    max_trials=100,\n",
    "    executions_per_trial=2,\n",
    "    directory=\"mnist_kt_test\",\n",
    "    overwrite=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "tuner.search_space_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
    "x_train = x_train.reshape((-1, 28 * 28)).astype(\"float32\") / 255\n",
    "x_test = x_test.reshape((-1, 28 * 28)).astype(\"float32\") / 255\n",
    "x_train_full = x_train[:]\n",
    "y_train_full = y_train[:]\n",
    "num_val_samples = 10000\n",
    "x_train, x_val = x_train[:-num_val_samples], x_train[-num_val_samples:]\n",
    "y_train, y_val = y_train[:-num_val_samples], y_train[-num_val_samples:]\n",
    "callbacks = [\n",
    "    keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5),\n",
    "]\n",
    "tuner.search(\n",
    "    x_train, y_train,\n",
    "    batch_size=128,\n",
    "    epochs=100,\n",
    "    validation_data=(x_val, y_val),\n",
    "    callbacks=callbacks,\n",
    "    verbose=2,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Querying the best hyperparameter configurations**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "top_n = 4\n",
    "best_hps = tuner.get_best_hyperparameters(top_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def get_best_epoch(hp):\n",
    "    model = build_model(hp)\n",
    "    callbacks=[\n",
    "        keras.callbacks.EarlyStopping(\n",
    "            monitor=\"val_loss\", mode=\"min\", patience=10)\n",
    "    ]\n",
    "    history = model.fit(\n",
    "        x_train, y_train,\n",
    "        validation_data=(x_val, y_val),\n",
    "        epochs=100,\n",
    "        batch_size=128,\n",
    "        callbacks=callbacks)\n",
    "    val_loss_per_epoch = history.history[\"val_loss\"]\n",
    "    best_epoch = val_loss_per_epoch.index(min(val_loss_per_epoch)) + 1\n",
    "    print(f\"Best epoch: {best_epoch}\")\n",
    "    return best_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def get_best_trained_model(hp):\n",
    "    best_epoch = get_best_epoch(hp)\n",
    "    model = build_model(hp)\n",
    "    model.fit(\n",
    "        x_train_full, y_train_full,\n",
    "        batch_size=128, epochs=int(best_epoch * 1.2))\n",
    "    return model\n",
    "\n",
    "best_models = []\n",
    "for hp in best_hps:\n",
    "    model = get_best_trained_model(hp)\n",
    "    model.evaluate(x_test, y_test)\n",
    "    best_models.append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "best_models = tuner.get_best_models(top_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### The art of crafting the right search space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### The future of hyperparameter tuning: automated machine learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Model ensembling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Scaling-up model training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Speeding up training on GPU with mixed precision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Understanding floating-point precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "np_array = np.zeros((2, 2))\n",
    "tf_tensor = tf.convert_to_tensor(np_array)\n",
    "tf_tensor.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "np_array = np.zeros((2, 2))\n",
    "tf_tensor = tf.convert_to_tensor(np_array, dtype=\"float32\")\n",
    "tf_tensor.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Mixed-precision training in practice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "keras.mixed_precision.set_global_policy(\"mixed_float16\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Multi-GPU training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Getting your hands on two or more GPUs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Single-host, multi-device synchronous training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### TPU training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Using a TPU via Google Colab"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Leveraging step fusing to improve TPU utilization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Summary"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "chapter13_best-practices-for-the-real-world.i",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}