{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "This is a companion notebook for the book [Deep Learning with Python, Third Edition](https://www.manning.com/books/deep-learning-with-python-third-edition). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n\n**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n\nThe book's contents are available online at [deeplearningwithpython.io](https://deeplearningwithpython.io)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!pip install keras keras-hub --upgrade -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"jax\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "cellView": "form",
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# @title\n",
    "import os\n",
    "from IPython.core.magic import register_cell_magic\n",
    "\n",
    "@register_cell_magic\n",
    "def backend(line, cell):\n",
    "    current, required = os.environ.get(\"KERAS_BACKEND\", \"\"), line.split()[-1]\n",
    "    if current == required:\n",
    "        get_ipython().run_cell(cell)\n",
    "    else:\n",
    "        print(\n",
    "            f\"This cell requires the {required} backend. To run it, change KERAS_BACKEND to \"\n",
    "            f\"\\\"{required}\\\" at the top of the notebook, restart the runtime, and rerun the notebook.\"\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Timeseries forecasting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Different kinds of timeseries tasks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### A temperature forecasting example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!wget https://s3.amazonaws.com/keras-datasets/jena_climate_2009_2016.csv.zip\n",
    "!unzip jena_climate_2009_2016.csv.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "fname = os.path.join(\"jena_climate_2009_2016.csv\")\n",
    "\n",
    "with open(fname) as f:\n",
    "    data = f.read()\n",
    "\n",
    "lines = data.split(\"\\n\")\n",
    "header = lines[0].split(\",\")\n",
    "lines = lines[1:]\n",
    "print(header)\n",
    "print(len(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "temperature = np.zeros((len(lines),))\n",
    "raw_data = np.zeros((len(lines), len(header) - 1))\n",
    "\n",
    "for i, line in enumerate(lines):\n",
    "    values = [float(x) for x in line.split(\",\")[1:]]\n",
    "    temperature[i] = values[1]\n",
    "    raw_data[i, :] = values[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "plt.plot(range(len(temperature)), temperature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "plt.plot(range(1440), temperature[:1440])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "num_train_samples = int(0.5 * len(raw_data))\n",
    "num_val_samples = int(0.25 * len(raw_data))\n",
    "num_test_samples = len(raw_data) - num_train_samples - num_val_samples\n",
    "print(\"num_train_samples:\", num_train_samples)\n",
    "print(\"num_val_samples:\", num_val_samples)\n",
    "print(\"num_test_samples:\", num_test_samples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Preparing the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "mean = raw_data[:num_train_samples].mean(axis=0)\n",
    "raw_data -= mean\n",
    "std = raw_data[:num_train_samples].std(axis=0)\n",
    "raw_data /= std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import keras\n",
    "\n",
    "int_sequence = np.arange(10)\n",
    "dummy_dataset = keras.utils.timeseries_dataset_from_array(\n",
    "    data=int_sequence[:-3],\n",
    "    targets=int_sequence[3:],\n",
    "    sequence_length=3,\n",
    "    batch_size=2,\n",
    ")\n",
    "\n",
    "for inputs, targets in dummy_dataset:\n",
    "    for i in range(inputs.shape[0]):\n",
    "        print([int(x) for x in inputs[i]], int(targets[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "sampling_rate = 6\n",
    "sequence_length = 120\n",
    "delay = sampling_rate * (sequence_length + 24 - 1)\n",
    "batch_size = 256\n",
    "\n",
    "train_dataset = keras.utils.timeseries_dataset_from_array(\n",
    "    raw_data[:-delay],\n",
    "    targets=temperature[delay:],\n",
    "    sampling_rate=sampling_rate,\n",
    "    sequence_length=sequence_length,\n",
    "    shuffle=True,\n",
    "    batch_size=batch_size,\n",
    "    start_index=0,\n",
    "    end_index=num_train_samples,\n",
    ")\n",
    "\n",
    "val_dataset = keras.utils.timeseries_dataset_from_array(\n",
    "    raw_data[:-delay],\n",
    "    targets=temperature[delay:],\n",
    "    sampling_rate=sampling_rate,\n",
    "    sequence_length=sequence_length,\n",
    "    shuffle=True,\n",
    "    batch_size=batch_size,\n",
    "    start_index=num_train_samples,\n",
    "    end_index=num_train_samples + num_val_samples,\n",
    ")\n",
    "\n",
    "test_dataset = keras.utils.timeseries_dataset_from_array(\n",
    "    raw_data[:-delay],\n",
    "    targets=temperature[delay:],\n",
    "    sampling_rate=sampling_rate,\n",
    "    sequence_length=sequence_length,\n",
    "    shuffle=True,\n",
    "    batch_size=batch_size,\n",
    "    start_index=num_train_samples + num_val_samples,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "for samples, targets in train_dataset:\n",
    "    print(\"samples shape:\", samples.shape)\n",
    "    print(\"targets shape:\", targets.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A commonsense, non-machine-learning baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def evaluate_naive_method(dataset):\n",
    "    total_abs_err = 0.0\n",
    "    samples_seen = 0\n",
    "    for samples, targets in dataset:\n",
    "        preds = samples[:, -1, 1] * std[1] + mean[1]\n",
    "        total_abs_err += np.sum(np.abs(preds - targets))\n",
    "        samples_seen += samples.shape[0]\n",
    "    return total_abs_err / samples_seen\n",
    "\n",
    "print(f\"Validation MAE: {evaluate_naive_method(val_dataset):.2f}\")\n",
    "print(f\"Test MAE: {evaluate_naive_method(test_dataset):.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Let's try a basic machine learning model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras import layers\n",
    "\n",
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.Flatten()(inputs)\n",
    "x = layers.Dense(16, activation=\"relu\")(x)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\"jena_dense.keras\", save_best_only=True)\n",
    "]\n",
    "model.compile(optimizer=\"adam\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(\n",
    "    train_dataset,\n",
    "    epochs=10,\n",
    "    validation_data=val_dataset,\n",
    "    callbacks=callbacks,\n",
    ")\n",
    "\n",
    "model = keras.models.load_model(\"jena_dense.keras\")\n",
    "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "loss = history.history[\"mae\"]\n",
    "val_loss = history.history[\"val_mae\"]\n",
    "epochs = range(1, len(loss) + 1)\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, \"r--\", label=\"Training MAE\")\n",
    "plt.plot(epochs, val_loss, \"b\", label=\"Validation MAE\")\n",
    "plt.title(\"Training and validation MAE\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Let's try a 1D convolutional model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.Conv1D(8, 24, activation=\"relu\")(inputs)\n",
    "x = layers.MaxPooling1D(2)(x)\n",
    "x = layers.Conv1D(8, 12, activation=\"relu\")(x)\n",
    "x = layers.MaxPooling1D(2)(x)\n",
    "x = layers.Conv1D(8, 6, activation=\"relu\")(x)\n",
    "x = layers.GlobalAveragePooling1D()(x)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\"jena_conv.keras\", save_best_only=True)\n",
    "]\n",
    "model.compile(optimizer=\"adam\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(\n",
    "    train_dataset,\n",
    "    epochs=10,\n",
    "    validation_data=val_dataset,\n",
    "    callbacks=callbacks,\n",
    ")\n",
    "\n",
    "model = keras.models.load_model(\"jena_conv.keras\")\n",
    "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Recurrent neural networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.LSTM(16)(inputs)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\"jena_lstm.keras\", save_best_only=True)\n",
    "]\n",
    "model.compile(optimizer=\"adam\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(\n",
    "    train_dataset,\n",
    "    epochs=10,\n",
    "    validation_data=val_dataset,\n",
    "    callbacks=callbacks,\n",
    ")\n",
    "\n",
    "model = keras.models.load_model(\"jena_lstm.keras\")\n",
    "print(\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Understanding recurrent neural networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "timesteps = 100\n",
    "input_features = 32\n",
    "output_features = 64\n",
    "inputs = np.random.random((timesteps, input_features))\n",
    "state_t = np.zeros((output_features,))\n",
    "W = np.random.random((output_features, input_features))\n",
    "U = np.random.random((output_features, output_features))\n",
    "b = np.random.random((output_features,))\n",
    "successive_outputs = []\n",
    "for input_t in inputs:\n",
    "    output_t = np.tanh(np.dot(W, input_t) + np.dot(U, state_t) + b)\n",
    "    successive_outputs.append(output_t)\n",
    "    state_t = output_t\n",
    "final_output_sequence = np.concatenate(successive_outputs, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A recurrent layer in Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "num_features = 14\n",
    "inputs = keras.Input(shape=(None, num_features))\n",
    "outputs = layers.SimpleRNN(16)(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "num_features = 14\n",
    "steps = 120\n",
    "inputs = keras.Input(shape=(steps, num_features))\n",
    "outputs = layers.SimpleRNN(16, return_sequences=False)(inputs)\n",
    "print(outputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "num_features = 14\n",
    "steps = 120\n",
    "inputs = keras.Input(shape=(steps, num_features))\n",
    "outputs = layers.SimpleRNN(16, return_sequences=True)(inputs)\n",
    "print(outputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(steps, num_features))\n",
    "x = layers.SimpleRNN(16, return_sequences=True)(inputs)\n",
    "x = layers.SimpleRNN(16, return_sequences=True)(x)\n",
    "outputs = layers.SimpleRNN(16)(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Getting the most out of recurrent neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Using recurrent dropout to fight overfitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.LSTM(32, recurrent_dropout=0.25)(inputs)\n",
    "x = layers.Dropout(0.5)(x)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\n",
    "        \"jena_lstm_dropout.keras\", save_best_only=True\n",
    "    )\n",
    "]\n",
    "model.compile(optimizer=\"adam\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(\n",
    "    train_dataset,\n",
    "    epochs=50,\n",
    "    validation_data=val_dataset,\n",
    "    callbacks=callbacks,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Stacking recurrent layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.GRU(32, recurrent_dropout=0.5, return_sequences=True)(inputs)\n",
    "x = layers.GRU(32, recurrent_dropout=0.5)(x)\n",
    "x = layers.Dropout(0.5)(x)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\n",
    "        \"jena_stacked_gru_dropout.keras\", save_best_only=True\n",
    "    )\n",
    "]\n",
    "model.compile(optimizer=\"adam\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(\n",
    "    train_dataset,\n",
    "    epochs=50,\n",
    "    validation_data=val_dataset,\n",
    "    callbacks=callbacks,\n",
    ")\n",
    "model = keras.models.load_model(\"jena_stacked_gru_dropout.keras\")\n",
    "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Using bidirectional RNNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.Bidirectional(layers.LSTM(16))(inputs)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "model.compile(optimizer=\"adam\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(\n",
    "    train_dataset,\n",
    "    epochs=10,\n",
    "    validation_data=val_dataset,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Going even further"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "chapter13_timeseries-forecasting",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}