soil-moisture.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98f8129e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import datetime\n",
    "\n",
    "import IPython\n",
    "import IPython.display\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import tensorflow as tf\n",
    "\n",
    "mpl.rcParams['figure.figsize'] = (8, 6)\n",
    "mpl.rcParams['axes.grid'] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3054aa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prep_data_cpc():\n",
    "    #read file\n",
    "    csv_path=\"./cpc-2.csv\"\n",
    "    df = pd.read_csv(csv_path)\n",
    "\n",
    "    #fill missing values\n",
    "    doy = df.pop('doy')\n",
    "    df['doys'] = np.sin((doy * 2 * np.pi)/365.24)\n",
    "    df['doyc'] = np.cos((doy * 2 * np.pi)/365.24)\n",
    "    sm = df.pop('sm')\n",
    "    sm = sm / 1000.0\n",
    "    df['sm'] = sm\n",
    "    lai = df.pop('lai')\n",
    "    lai = lai / 4\n",
    "    df['lai'] = lai\n",
    "    ir = df.pop('ir')\n",
    "    ir_mean = ir.mean()\n",
    "    ir_std = ir.std()\n",
    "    ir = (ir - ir_mean) / ir_std\n",
    "    df['ir'] = ir\n",
    "    \n",
    "    return df\n",
    "\n",
    "def prep_data_uscrn():\n",
    "    #read file\n",
    "    csv_path=\"./uscrn-2.csv\"\n",
    "    df = pd.read_csv(csv_path)\n",
    "    \n",
    "    #fill missing values\n",
    "    for i in range(1,6):\n",
    "        sm = df['sm'+str(i)]\n",
    "        bad_sm1 = (sm < 0)\n",
    "        bad_sm2 = (sm > 1)\n",
    "        sm[bad_sm1] = 0\n",
    "        sm[bad_sm2] = 0\n",
    "    df['sm']=df[['sm1','sm2','sm3','sm4','sm5']].apply(np.max,axis=1)\n",
    "    for i in range(1,6):\n",
    "        df.pop('sm'+str(i))\n",
    "    df = df[df.sm != 0]\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    doy = df.pop('doy')\n",
    "    df['doys'] = np.sin((doy * 2 * np.pi)/365.24)\n",
    "    df['doyc'] = np.cos((doy * 2 * np.pi)/365.24)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54f868fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = prep_data_uscrn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "019d1b48",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "968fc19a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.describe(include='all').transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c099bf5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#plot data\n",
    "s = 15*1817\n",
    "e = s + 3*1817\n",
    "plot_features = df[:][s:e]\n",
    "plot_features.index = range(0,e-s)\n",
    "_ = plot_features.plot(subplots=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbd4b24c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = pd.get_dummies(df, columns=['lu','st','cd'], drop_first=True)\n",
    "df = pd.get_dummies(df, columns=['station'], drop_first=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3636b415",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_fft(plot_col='sm', ylim=None):\n",
    "    fft = tf.signal.rfft(df[plot_col])\n",
    "    f_per_dataset = np.arange(0, len(fft))\n",
    "\n",
    "    n_samples_d = len(df[plot_col])\n",
    "    days_per_year = 365.2524\n",
    "    years_per_dataset = n_samples_d/(days_per_year)\n",
    "\n",
    "    f_per_year = f_per_dataset/years_per_dataset\n",
    "    plt.figure()\n",
    "    plt.step(f_per_year, np.abs(fft))\n",
    "    plt.xscale('log')\n",
    "    plt.title(plot_col)\n",
    "    if ylim is not None:\n",
    "        plt.ylim(0, ylim)\n",
    "    plt.xlim([0.1, max(plt.xlim())])\n",
    "#     plt.xticks([0.25, 1], labels=['1/4-Years', '1/Year'])\n",
    "    plt.xticks([0.2, 1], labels=['1/5-Years', '1/Year'])\n",
    "    _ = plt.xlabel('Frequency (log scale)')\n",
    "\n",
    "plot_fft(plot_col='sm', ylim=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f25ebe63",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set prediction point (e.g. tomorrow = 1 day shift)\n",
    "PRED_SHIFT=1\n",
    "#how much to shift the input\n",
    "INPUT_SHIFT=(0 if PRED_SHIFT > 0 else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80358653",
   "metadata": {},
   "outputs": [],
   "source": [
    "column_indices = {name: i for i, name in enumerate(df.columns)}\n",
    "n = len(df)\n",
    "train_df = df[0:int(n*0.7)]\n",
    "val_df = df[int(n*0.7):int(n*0.9)]\n",
    "test_df = df[int(n*0.9):]\n",
    "num_features = df.shape[1]\n",
    "sm_index = column_indices['sm']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5658533",
   "metadata": {},
   "outputs": [],
   "source": [
    "class WindowGenerator():\n",
    "  def __init__(self, input_width, label_width, shift,\n",
    "               train_df=train_df, val_df=val_df, test_df=test_df,\n",
    "               label_columns=None):\n",
    "    # Store the raw data.\n",
    "    self.train_df = train_df\n",
    "    self.val_df = val_df\n",
    "    self.test_df = test_df\n",
    "\n",
    "    # Work out the label column indices.\n",
    "    self.label_columns = label_columns\n",
    "    if label_columns is not None:\n",
    "      self.label_columns_indices = {name: i for i, name in\n",
    "                                    enumerate(label_columns)}\n",
    "    self.column_indices = {name: i for i, name in\n",
    "                           enumerate(train_df.columns)}\n",
    "\n",
    "    # Work out the window parameters.\n",
    "    self.input_width = input_width\n",
    "    self.label_width = label_width\n",
    "    self.shift = shift\n",
    "\n",
    "    self.total_window_size = input_width + shift\n",
    "\n",
    "    self.input_slice = slice(0, input_width)\n",
    "    self.input_indices = np.arange(self.total_window_size)[self.input_slice]\n",
    "\n",
    "    self.label_start = self.total_window_size - self.label_width\n",
    "    self.labels_slice = slice(self.label_start, None)\n",
    "    self.label_indices = np.arange(self.total_window_size)[self.labels_slice]\n",
    "\n",
    "  def __repr__(self):\n",
    "    return '\\n'.join([\n",
    "        f'Total window size: {self.total_window_size}',\n",
    "        f'Input indices: {self.input_indices}',\n",
    "        f'Label indices: {self.label_indices}',\n",
    "        f'Label column name(s): {self.label_columns}'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a31cb51c",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2 = WindowGenerator(input_width=7, label_width=1, shift=PRED_SHIFT,\n",
    "                     label_columns=['sm'])\n",
    "w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c893f36b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_window(self, features):\n",
    "  inputs = features[:, self.input_slice, :]\n",
    "  labels = features[:, self.labels_slice, :]\n",
    "  if self.label_columns is not None:\n",
    "    labels = tf.stack(\n",
    "        [labels[:, :, self.column_indices[name]] for name in self.label_columns],\n",
    "        axis=-1)\n",
    "\n",
    "  # Slicing doesn't preserve static shape information, so set the shapes\n",
    "  # manually. This way the `tf.data.Datasets` are easier to inspect.\n",
    "  inputs.set_shape([None, self.input_width, None])\n",
    "  labels.set_shape([None, self.label_width, None])\n",
    "    \n",
    "  #set \"sm\" to 0 for the last-time step when SHIFT=0\n",
    "  if PRED_SHIFT == 0:\n",
    "      mask = np.ones(shape=inputs.shape.as_list()[1:])\n",
    "      mask[self.input_width - 1, sm_index] = 0\n",
    "      inputs = tf.multiply(inputs, mask)\n",
    "\n",
    "  return inputs, labels\n",
    "\n",
    "WindowGenerator.split_window = split_window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06e9e5ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stack three slices, the length of the total window.\n",
    "shift = 100000\n",
    "example_window = tf.stack([np.array(train_df[shift:shift+w2.total_window_size]),\n",
    "                           np.array(train_df[shift+100:shift+100+w2.total_window_size]),\n",
    "                           np.array(train_df[shift+200:shift+200+w2.total_window_size])])\n",
    "\n",
    "example_inputs, example_labels = w2.split_window(example_window)\n",
    "\n",
    "print('All shapes are: (batch, time, features)')\n",
    "print(f'Window shape: {example_window.shape}')\n",
    "print(f'Inputs shape: {example_inputs.shape}')\n",
    "print(f'Labels shape: {example_labels.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ee687b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2.example = example_inputs, example_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "887e3ac0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(self, model=None, plot_col='sm', max_subplots=3):\n",
    "  inputs, labels = self.example\n",
    "  plt.figure(figsize=(12, 8))\n",
    "  plot_col_index = self.column_indices[plot_col]\n",
    "  max_n = min(max_subplots, len(inputs))\n",
    "  for n in range(max_n):\n",
    "    plt.subplot(max_n, 1, n+1)\n",
    "    if plot_col == 'sm':\n",
    "        plt.ylabel(f'{plot_col} m/m')\n",
    "        if PRED_SHIFT == 0:\n",
    "            end = -1\n",
    "        else:\n",
    "            end = None\n",
    "        plt.plot(self.input_indices[:end], inputs[n, :end, plot_col_index],\n",
    "             label='Inputs', marker='.', zorder=-10)\n",
    "    else:\n",
    "        plt.ylabel(f'{plot_col}')\n",
    "        plt.plot(self.input_indices[:], inputs[n, :, plot_col_index],\n",
    "             label='Inputs', marker='.', zorder=-10)\n",
    "\n",
    "    if self.label_columns:\n",
    "      label_col_index = self.label_columns_indices.get(plot_col, None)\n",
    "    else:\n",
    "      label_col_index = plot_col_index\n",
    "\n",
    "    if label_col_index is None:\n",
    "      continue\n",
    "\n",
    "    plt.scatter(self.label_indices, labels[n, :, label_col_index],\n",
    "                edgecolors='k', label='Labels', c='#2ca02c', s=64)\n",
    "    if model is not None:\n",
    "      predictions = model(inputs)\n",
    "      plt.scatter(self.label_indices, predictions[n, :, label_col_index],\n",
    "                  marker='X', edgecolors='k', label='Predictions',\n",
    "                  c='#ff7f0e', s=64)\n",
    "\n",
    "    if n == 0:\n",
    "      plt.legend()\n",
    "\n",
    "  plt.xlabel('Days')\n",
    "\n",
    "WindowGenerator.plot = plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7b4640d",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ca05915",
   "metadata": {},
   "outputs": [],
   "source": [
    "#w2.plot(plot_col='ir')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8437c65d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_dataset(self, data):\n",
    "  data = np.array(data, dtype=np.float32)\n",
    "  ds = tf.keras.utils.timeseries_dataset_from_array(\n",
    "      data=data,\n",
    "      targets=None,\n",
    "      sequence_length=self.total_window_size,\n",
    "      sequence_stride=1,\n",
    "      shuffle=True,\n",
    "      batch_size=32,)\n",
    "\n",
    "  ds = ds.map(self.split_window)\n",
    "\n",
    "  return ds\n",
    "\n",
    "WindowGenerator.make_dataset = make_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7c807e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "@property\n",
    "def train(self):\n",
    "  return self.make_dataset(self.train_df)\n",
    "\n",
    "@property\n",
    "def val(self):\n",
    "  return self.make_dataset(self.val_df)\n",
    "\n",
    "@property\n",
    "def test(self):\n",
    "  return self.make_dataset(self.test_df)\n",
    "\n",
    "@property\n",
    "def example(self):\n",
    "  \"\"\"Get and cache an example batch of `inputs, labels` for plotting.\"\"\"\n",
    "  result = getattr(self, '_example', None)\n",
    "  if result is None:\n",
    "    # No example batch was found, so get one from the `.train` dataset\n",
    "    result = next(iter(self.train))\n",
    "    # And cache it for next time\n",
    "    self._example = result\n",
    "  return result\n",
    "\n",
    "WindowGenerator.train = train\n",
    "WindowGenerator.val = val\n",
    "WindowGenerator.test = test\n",
    "WindowGenerator.example = example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4fc2dde",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Each element is an (inputs, label) pair.\n",
    "w2.train.element_spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30f3feeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "for example_inputs, example_labels in w2.train.take(1):\n",
    "  print(f'Inputs shape (batch, time, features): {example_inputs.shape}')\n",
    "  print(f'Labels shape (batch, time, features): {example_labels.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0caa89af",
   "metadata": {},
   "outputs": [],
   "source": [
    "single_step_window = WindowGenerator(\n",
    "    input_width=1+INPUT_SHIFT, label_width=1, shift=PRED_SHIFT,\n",
    "    label_columns=['sm'])\n",
    "single_step_window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5a51761",
   "metadata": {},
   "outputs": [],
   "source": [
    "for example_inputs, example_labels in single_step_window.train.take(1):\n",
    "  print(f'Inputs shape (batch, time, features): {example_inputs.shape}')\n",
    "  print(f'Labels shape (batch, time, features): {example_labels.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cfd5b6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_EPOCHS = 20\n",
    "\n",
    "def compile_and_fit(model, window, patience=2):\n",
    "  early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                                                patience=patience,\n",
    "                                                mode='min')\n",
    "  \n",
    "\n",
    "  model.compile(loss=tf.losses.MeanSquaredError(),\n",
    "                optimizer=tf.optimizers.Adam(),\n",
    "                metrics=[tf.metrics.MeanAbsoluteError()])\n",
    "\n",
    "  history = model.fit(window.train, epochs=MAX_EPOCHS,\n",
    "                      validation_data=window.val,\n",
    "                      callbacks=[early_stopping])\n",
    "  return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d969144d",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_performance = {}\n",
    "performance = {}\n",
    "np.random.seed(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "deae43df",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualWrapper(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    paddings = tf.constant([[0,0],[INPUT_SHIFT,0]])\n",
    "    res = tf.pad(res, paddings, \"SYMMETRIC\")\n",
    "    res = tf.roll(res, shift=INPUT_SHIFT, axis=1)\n",
    "    \n",
    "    return res[:, INPUT_SHIFT:, tf.newaxis] + delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8033b689",
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline = ResidualWrapper(\n",
    "    tf.keras.layers.Lambda(lambda x: 0.0)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e5daa8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(baseline, single_step_window)\n",
    "\n",
    "val_performance['Baseline'] = baseline.evaluate(single_step_window.val)\n",
    "performance['Baseline'] = baseline.evaluate(single_step_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5be8978a",
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_window = WindowGenerator(\n",
    "    input_width=24, label_width=24, shift=PRED_SHIFT,\n",
    "    label_columns=['sm'])\n",
    "\n",
    "wide_window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d66cd422",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Input shape:', wide_window.example[0].shape)\n",
    "print('Output shape:', baseline(wide_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03b9b24d",
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_window.plot(baseline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e04d96d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#wide_window.plot(baseline, plot_col='lai')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f837df3",
   "metadata": {},
   "outputs": [],
   "source": [
    "linear = ResidualWrapper(\n",
    "    tf.keras.Sequential([\n",
    "        tf.keras.layers.Dense(units=1, kernel_initializer=tf.initializers.zeros(),\n",
    "                              kernel_regularizer=tf.keras.regularizers.l2(1e-4))\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efcb16b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Input shape:', single_step_window.example[0].shape)\n",
    "print('Output shape:', linear(single_step_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1239256",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(linear, single_step_window)\n",
    "\n",
    "val_performance['Linear'] = linear.evaluate(single_step_window.val)\n",
    "performance['Linear'] = linear.evaluate(single_step_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9973f3a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Input shape:', wide_window.example[0].shape)\n",
    "print('Output shape:', baseline(wide_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99757147",
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_window.plot(linear)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d514135",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(x = range(len(train_df.columns)),\n",
    "        height=linear.model.layers[0].kernel[:,0].numpy())\n",
    "axis = plt.gca()\n",
    "axis.set_xticks(range(len(train_df.columns)))\n",
    "_ = axis.set_xticklabels(train_df.columns, rotation=90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ee2677c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dense =  ResidualWrapper(\n",
    "    tf.keras.Sequential([\n",
    "        tf.keras.layers.Dense(units=64, activation='relu',kernel_regularizer=tf.keras.regularizers.l2(1e-4)),\n",
    "        tf.keras.layers.Dense(units=64, activation='relu',kernel_regularizer=tf.keras.regularizers.l2(1e-4)),\n",
    "        tf.keras.layers.Dense(units=1, kernel_regularizer=tf.keras.regularizers.l2(1e-4),\n",
    "                              kernel_initializer=tf.initializers.zeros())\n",
    "    ])\n",
    ")\n",
    "history = compile_and_fit(dense, single_step_window)\n",
    "\n",
    "val_performance['Dense'] = dense.evaluate(single_step_window.val)\n",
    "performance['Dense'] = dense.evaluate(single_step_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a46bc1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_window.plot(dense)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cae284ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualWrapper(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    paddings = tf.constant([[0,0],[1+INPUT_SHIFT,0]])\n",
    "    res = tf.pad(res, paddings, \"SYMMETRIC\")\n",
    "    res1 = tf.roll(res, shift=INPUT_SHIFT, axis=1)\n",
    "    res2 = tf.roll(res, shift=1+INPUT_SHIFT, axis=1)\n",
    "    res = 2 * res1 - res2\n",
    "    res = res[:, 1+INPUT_SHIFT:, tf.newaxis]\n",
    "    return  res + delta\n",
    "\n",
    "class ResidualWrapperSame(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    paddings = tf.constant([[0,0],[INPUT_SHIFT,0]])\n",
    "    res = tf.pad(res, paddings, \"SYMMETRIC\")\n",
    "    res = tf.roll(res, shift=INPUT_SHIFT, axis=1)\n",
    "    res = res[:, INPUT_SHIFT:, tf.newaxis]\n",
    "    return  res + delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "744fdec4",
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_multi = ResidualWrapperSame(\n",
    "    tf.keras.layers.Lambda(lambda x: 0.0)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9428d62",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(baseline_multi, wide_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "val_performance['Baseline multi'] = baseline_multi.evaluate(wide_window.val)\n",
    "performance['Baseline multi'] = baseline_multi.evaluate(wide_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7423ae6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Input shape:', wide_window.example[0].shape)\n",
    "print('Output shape:', baseline_multi(wide_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f0116b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_window.plot(baseline_multi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88e74695",
   "metadata": {},
   "outputs": [],
   "source": [
    "# MULTI-STEP model\n",
    "CONV_WIDTH = 3\n",
    "conv_window = WindowGenerator(\n",
    "    input_width=CONV_WIDTH,\n",
    "    label_width=1,\n",
    "    shift=PRED_SHIFT,\n",
    "    label_columns=['sm'])\n",
    "\n",
    "conv_window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f144724",
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_window.plot()\n",
    "plt.title(f\"Given {CONV_WIDTH} days of inputs, predict 1 day into the future.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83047587",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualWrapper(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    paddings = tf.constant([[0,0],[1+INPUT_SHIFT,0]])\n",
    "    res = tf.pad(res, paddings, \"SYMMETRIC\")\n",
    "    res1 = tf.roll(res, shift=INPUT_SHIFT, axis=1)\n",
    "    res2 = tf.roll(res, shift=1+INPUT_SHIFT, axis=1)\n",
    "    res = 2 * res1 - res2\n",
    "    res = res[:, -1:, tf.newaxis]\n",
    "    return  res + delta\n",
    "class ResidualWrapperSame(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    paddings = tf.constant([[0,0],[INPUT_SHIFT,0]])\n",
    "    res = tf.pad(res, paddings, \"SYMMETRIC\")\n",
    "    res = tf.roll(res, shift=INPUT_SHIFT, axis=1)\n",
    "    res = res[:, -1:, tf.newaxis]\n",
    "    return  res + delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef748db7",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_step_dense = ResidualWrapperSame(\n",
    "    tf.keras.Sequential([\n",
    "        # Shape: (time, features) => (time*features)\n",
    "        tf.keras.layers.Flatten(),\n",
    "        tf.keras.layers.Dense(units=32, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(1e-4)),\n",
    "        tf.keras.layers.Dense(units=32, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(1e-4)),\n",
    "        tf.keras.layers.Dense(units=1, kernel_initializer=tf.initializers.zeros(), \n",
    "                              kernel_regularizer=tf.keras.regularizers.l2(1e-4)),\n",
    "        # Add back the time dimension.\n",
    "        # Shape: (outputs) => (1, outputs)\n",
    "        tf.keras.layers.Reshape([1, -1]),\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30e50d8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Input shape:', conv_window.example[0].shape)\n",
    "print('Output shape:', multi_step_dense(conv_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "163654a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(multi_step_dense, conv_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "val_performance['Multi step dense'] = multi_step_dense.evaluate(conv_window.val)\n",
    "performance['Multi step dense'] = multi_step_dense.evaluate(conv_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "473d9467",
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_window.plot(multi_step_dense)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6985f1b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualWrapper(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    res1 = tf.roll(res, shift=INPUT_SHIFT, axis=1)\n",
    "    res2 = tf.roll(res, shift=1+INPUT_SHIFT, axis=1)\n",
    "    res = 2 * res1 - res2\n",
    "    res = res[:, (CONV_WIDTH - 1):, tf.newaxis]\n",
    "    return res + delta\n",
    "class ResidualWrapperSame(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    paddings = tf.constant([[0,0],[INPUT_SHIFT,0]])\n",
    "    res = tf.pad(res, paddings, \"SYMMETRIC\")\n",
    "    res = tf.roll(res, shift=INPUT_SHIFT, axis=1)\n",
    "    res = res[:, (CONV_WIDTH-1):, tf.newaxis]\n",
    "    return  res + delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7a08c23",
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_model = ResidualWrapperSame(\n",
    "    tf.keras.Sequential([\n",
    "        tf.keras.layers.Conv1D(filters=32,\n",
    "                               kernel_size=(CONV_WIDTH,),\n",
    "                               activation='relu',\n",
    "                               kernel_regularizer=tf.keras.regularizers.l2(1e-4)),\n",
    "        tf.keras.layers.Dense(units=32, activation='relu', \n",
    "                              kernel_regularizer=tf.keras.regularizers.l2(1e-4)),\n",
    "        tf.keras.layers.Dense(units=1, kernel_initializer=tf.initializers.zeros(),\n",
    "                              kernel_regularizer=tf.keras.regularizers.l2(1e-4)),\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21cf3c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Conv window\")\n",
    "print('Input shape:', conv_window.example[0].shape)\n",
    "print('Labels shape:', conv_window.example[1].shape)\n",
    "print('Output shape:', conv_model(conv_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "304777cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Wide window\")\n",
    "print('Input shape:', wide_window.example[0].shape)\n",
    "print('Labels shape:', wide_window.example[1].shape)\n",
    "print('Output shape:', conv_model(wide_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16e689f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "LABEL_WIDTH = 24\n",
    "INPUT_WIDTH = LABEL_WIDTH + (CONV_WIDTH - 1)\n",
    "wide_conv_window = WindowGenerator(\n",
    "    input_width=INPUT_WIDTH,\n",
    "    label_width=LABEL_WIDTH,\n",
    "    shift=PRED_SHIFT,\n",
    "    label_columns=['sm'])\n",
    "\n",
    "wide_conv_window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0b18c92",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Wide conv window\")\n",
    "print('Input shape:', wide_conv_window.example[0].shape)\n",
    "print('Labels shape:', wide_conv_window.example[1].shape)\n",
    "print('Output shape:', conv_model(wide_conv_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b53dd7a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(conv_model, wide_conv_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "val_performance['Conv'] = conv_model.evaluate(wide_conv_window.val)\n",
    "performance['Conv'] = conv_model.evaluate(wide_conv_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "115fcf58",
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_window.plot(conv_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "051e1638",
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_conv_window.plot(conv_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af8f20ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "#wide_conv_window.plot(conv_model, plot_col='gf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56e79991",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualWrapper(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    paddings = tf.constant([[0,0],[1+INPUT_SHIFT,0]])\n",
    "    res = tf.pad(res, paddings, \"SYMMETRIC\")\n",
    "    res1 = tf.roll(res, shift=INPUT_SHIFT, axis=1)\n",
    "    res2 = tf.roll(res, shift=1+INPUT_SHIFT, axis=1)\n",
    "    res = 2 * res1 - res2\n",
    "    res = res[:, 1+INPUT_SHIFT:, tf.newaxis]\n",
    "    return  res + delta\n",
    "class ResidualWrapperSame(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    paddings = tf.constant([[0,0],[INPUT_SHIFT,0]])\n",
    "    res = tf.pad(res, paddings, \"SYMMETRIC\")\n",
    "    res = tf.roll(res, shift=INPUT_SHIFT, axis=1)\n",
    "    res = res[:, INPUT_SHIFT:, tf.newaxis]\n",
    "    return  res + delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f543b032",
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_model = ResidualWrapperSame(\n",
    "    tf.keras.models.Sequential([\n",
    "        tf.keras.layers.LSTM(32, return_sequences=True,\n",
    "                             kernel_regularizer=tf.keras.regularizers.l2(1e-4)),\n",
    "        tf.keras.layers.Dense(units=1, kernel_regularizer=tf.keras.regularizers.l2(1e-4),\n",
    "                              kernel_initializer=tf.initializers.zeros())\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27abd968",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Wide window\")\n",
    "print('Input shape:', wide_window.example[0].shape)\n",
    "print('Labels shape:', wide_window.example[1].shape)\n",
    "print('Output shape:', lstm_model(wide_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30d93f29",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(lstm_model, wide_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "val_performance['LSTM'] = lstm_model.evaluate(wide_window.val)\n",
    "performance['LSTM'] = lstm_model.evaluate(wide_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03866478",
   "metadata": {},
   "outputs": [],
   "source": [
    "wide_window.plot(lstm_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3a06e33",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.arange(len(performance))\n",
    "width = 0.3\n",
    "metric_name = 'mean_absolute_error'\n",
    "metric_index = lstm_model.metrics_names.index('mean_absolute_error')\n",
    "val_mae = [v[metric_index] for v in val_performance.values()]\n",
    "test_mae = [v[metric_index] for v in performance.values()]\n",
    "\n",
    "plt.ylabel('mean_absolute_error [sm]')\n",
    "plt.bar(x - 0.17, val_mae, width, label='Validation')\n",
    "plt.bar(x + 0.17, test_mae, width, label='Test')\n",
    "plt.xticks(ticks=x, labels=performance.keys(),\n",
    "           rotation=45)\n",
    "_ = plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86fc215c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, value in performance.items():\n",
    "  print(f'{name:12s}: {value[1]:0.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3473553",
   "metadata": {},
   "outputs": [],
   "source": [
    "OUT_STEPS = 24\n",
    "multi_window = WindowGenerator(input_width=24,\n",
    "                               label_width=OUT_STEPS,\n",
    "                               shift=OUT_STEPS,\n",
    "                               label_columns=['sm'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29ca4cc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_window.plot()\n",
    "multi_window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d104866",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_val_performance = {}\n",
    "multi_performance = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8456ef00",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualWrapperLast(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, -1:, sm_index]\n",
    "    res = res[:, :, tf.newaxis]\n",
    "    res = tf.tile(res, [1, OUT_STEPS, 1])\n",
    "    return  res + delta\n",
    "\n",
    "class ResidualWrapperRepeat(tf.keras.Model):\n",
    "  def __init__(self, model):\n",
    "    super().__init__()\n",
    "    self.model = model\n",
    "\n",
    "  def call(self, inputs, *args, **kwargs):\n",
    "    delta = self.model(inputs, *args, **kwargs)\n",
    "\n",
    "    # The prediction for each time step is the input\n",
    "    # from the previous time step plus the delta\n",
    "    # calculated by the model.\n",
    "    res = inputs[:, :, sm_index]\n",
    "    res = res[:, :, tf.newaxis]\n",
    "    return  res + delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "592c6bc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_multi_output_last = ResidualWrapperLast(\n",
    "    tf.keras.layers.Lambda(lambda x: 0.0)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8dd0a24",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Multi window\")\n",
    "print('Input shape:', multi_window.example[0].shape)\n",
    "print('Labels shape:', multi_window.example[1].shape)\n",
    "print('Output shape:', baseline_multi_output_last(multi_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb1f4d44",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(baseline_multi_output_last, wide_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "multi_val_performance['Last'] = baseline_multi_output_last.evaluate(multi_window.val)\n",
    "multi_performance['Last'] = baseline_multi_output_last.evaluate(multi_window.test, verbose=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db417db3",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_window.plot(baseline_multi_output_last)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c4419b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_multi_output_repeat = ResidualWrapperRepeat(\n",
    "    tf.keras.layers.Lambda(lambda x: 0.0)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02800fb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(baseline_multi_output_repeat, wide_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "multi_val_performance['Repeat'] = baseline_multi_output_repeat.evaluate(multi_window.val)\n",
    "multi_performance['Repeat'] = baseline_multi_output_repeat.evaluate(multi_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7f44944",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_window.plot(baseline_multi_output_repeat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81cbf344",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_linear_model = tf.keras.Sequential([\n",
    "    # Take the last time-step.\n",
    "    # Shape [batch, time, features] => [batch, 1, features]\n",
    "    tf.keras.layers.Lambda(lambda x: x[:, -1:, :]),\n",
    "    # Shape => [batch, 1, out_steps*features]\n",
    "    tf.keras.layers.Dense(OUT_STEPS,\n",
    "                          kernel_initializer=tf.initializers.zeros()),\n",
    "    # Shape => [batch, out_steps, features]\n",
    "    tf.keras.layers.Reshape([OUT_STEPS, 1])\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d4ac41d",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(multi_linear_model, multi_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "multi_val_performance['Linear'] = multi_linear_model.evaluate(multi_window.val)\n",
    "multi_performance['Linear'] = multi_linear_model.evaluate(multi_window.test, verbose=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25927868",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_window.plot(multi_linear_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "003d9421",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_dense_model = tf.keras.Sequential([\n",
    "    # Take the last time step.\n",
    "    # Shape [batch, time, features] => [batch, 1, features]\n",
    "    tf.keras.layers.Lambda(lambda x: x[:, -1:, :]),\n",
    "    # Shape => [batch, 1, dense_units]\n",
    "    tf.keras.layers.Dense(512, activation='relu'),\n",
    "    # Shape => [batch, out_steps*features]\n",
    "    tf.keras.layers.Dense(OUT_STEPS,\n",
    "                          kernel_initializer=tf.initializers.zeros()),\n",
    "    # Shape => [batch, out_steps, features]\n",
    "    tf.keras.layers.Reshape([OUT_STEPS, 1])\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d1421b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(multi_dense_model, multi_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "multi_val_performance['Dense'] = multi_dense_model.evaluate(multi_window.val)\n",
    "multi_performance['Dense'] = multi_dense_model.evaluate(multi_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10a6be9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_window.plot(multi_dense_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d759c48",
   "metadata": {},
   "outputs": [],
   "source": [
    "CONV_WIDTH = 3\n",
    "multi_conv_model = tf.keras.Sequential([\n",
    "    # Shape [batch, time, features] => [batch, CONV_WIDTH, features]\n",
    "    tf.keras.layers.Lambda(lambda x: x[:, -CONV_WIDTH:, :]),\n",
    "    # Shape => [batch, 1, conv_units]\n",
    "    tf.keras.layers.Conv1D(256, activation='relu', kernel_size=(CONV_WIDTH)),\n",
    "    # Shape => [batch, 1,  out_steps*features]\n",
    "    tf.keras.layers.Dense(OUT_STEPS,\n",
    "                          kernel_initializer=tf.initializers.zeros()),\n",
    "    # Shape => [batch, out_steps, features]\n",
    "    tf.keras.layers.Reshape([OUT_STEPS, 1])\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7224994b",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(multi_conv_model, multi_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "\n",
    "multi_val_performance['Conv'] = multi_conv_model.evaluate(multi_window.val)\n",
    "multi_performance['Conv'] = multi_conv_model.evaluate(multi_window.test, verbose=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3710b84",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_window.plot(multi_conv_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de6bfa2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_lstm_model = tf.keras.Sequential([\n",
    "    # Shape [batch, time, features] => [batch, lstm_units].\n",
    "    # Adding more `lstm_units` just overfits more quickly.\n",
    "    tf.keras.layers.LSTM(32, return_sequences=False),\n",
    "    # Shape => [batch, out_steps*features].\n",
    "    tf.keras.layers.Dense(OUT_STEPS,\n",
    "                          kernel_initializer=tf.initializers.zeros()),\n",
    "    # Shape => [batch, out_steps, features].\n",
    "    tf.keras.layers.Reshape([OUT_STEPS, 1])\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bbbf131",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(multi_lstm_model, multi_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "\n",
    "multi_val_performance['LSTM'] = multi_lstm_model.evaluate(multi_window.val)\n",
    "multi_performance['LSTM'] = multi_lstm_model.evaluate(multi_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbd55895",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_window.plot(multi_lstm_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1a11029",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeedBack(tf.keras.Model):\n",
    "  def __init__(self, units, out_steps):\n",
    "    super().__init__()\n",
    "    self.out_steps = out_steps\n",
    "    self.units = units\n",
    "    self.lstm_cell = tf.keras.layers.LSTMCell(units)\n",
    "    # Also wrap the LSTMCell in an RNN to simplify the `warmup` method.\n",
    "    self.lstm_rnn = tf.keras.layers.RNN(self.lstm_cell, return_state=True)\n",
    "    self.dense = tf.keras.layers.Dense(num_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bea51cbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "feedback_model = FeedBack(units=32, out_steps=OUT_STEPS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b0d08f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def warmup(self, inputs):\n",
    "  # inputs.shape => (batch, time, features)\n",
    "  # x.shape => (batch, lstm_units)\n",
    "  x, *state = self.lstm_rnn(inputs)\n",
    "\n",
    "  # predictions.shape => (batch, features)\n",
    "  prediction = self.dense(x)\n",
    "  return prediction, state\n",
    "\n",
    "FeedBack.warmup = warmup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c7a6891",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction, state = feedback_model.warmup(multi_window.example[0])\n",
    "prediction.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "499a243f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def call(self, inputs, training=None):\n",
    "  # Use a TensorArray to capture dynamically unrolled outputs.\n",
    "  predictions = []\n",
    "  # Initialize the LSTM state.\n",
    "  prediction, state = self.warmup(inputs)\n",
    "\n",
    "  # Insert the first prediction.\n",
    "  predictions.append(prediction)\n",
    "\n",
    "  # Run the rest of the prediction steps.\n",
    "  for n in range(1, self.out_steps):\n",
    "    # Use the last prediction as input.\n",
    "    x = prediction\n",
    "    # Execute one lstm step.\n",
    "    x, state = self.lstm_cell(x, states=state,\n",
    "                              training=training)\n",
    "    # Convert the lstm output to a prediction.\n",
    "    prediction = self.dense(x)\n",
    "    # Add the prediction to the output.\n",
    "    predictions.append(prediction)\n",
    "\n",
    "  # predictions.shape => (time, batch, features)\n",
    "  predictions = tf.stack(predictions)\n",
    "  # predictions.shape => (batch, time, features)\n",
    "  predictions = tf.transpose(predictions, [1, 0, 2])\n",
    "  return predictions\n",
    "\n",
    "FeedBack.call = call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cc460ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Output shape (batch, time, features): ', feedback_model(multi_window.example[0]).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d752ea96",
   "metadata": {},
   "outputs": [],
   "source": [
    "history = compile_and_fit(feedback_model, multi_window)\n",
    "\n",
    "IPython.display.clear_output()\n",
    "\n",
    "multi_val_performance['AR LSTM'] = feedback_model.evaluate(multi_window.val)\n",
    "multi_performance['AR LSTM'] = feedback_model.evaluate(multi_window.test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a2a0be2",
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_window.plot(feedback_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae5cf877",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.arange(len(multi_performance))\n",
    "width = 0.3\n",
    "\n",
    "metric_name = 'mean_absolute_error'\n",
    "metric_index = lstm_model.metrics_names.index('mean_absolute_error')\n",
    "val_mae = [v[metric_index] for v in multi_val_performance.values()]\n",
    "test_mae = [v[metric_index] for v in multi_performance.values()]\n",
    "\n",
    "plt.bar(x - 0.17, val_mae, width, label='Validation')\n",
    "plt.bar(x + 0.17, test_mae, width, label='Test')\n",
    "plt.xticks(ticks=x, labels=multi_performance.keys(),\n",
    "           rotation=45)\n",
    "plt.ylabel(f'MAE (average over all times and outputs)')\n",
    "_ = plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23e6aa79",
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, value in multi_performance.items():\n",
    "  print(f'{name:8s}: {value[1]:0.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97642263",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}