10_introduction_to_artificial_neural_networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 10 – Introduction to Artificial Neural Networks**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code and solutions to the exercises in chapter 10._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ann\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perceptrons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "iris = load_iris()\n",
    "X = iris.data[:, (2, 3)]  # petal length, petal width\n",
    "y = (iris.target == 0).astype(np.int)\n",
    "\n",
    "per_clf = Perceptron(max_iter=100, random_state=42)\n",
    "per_clf.fit(X, y)\n",
    "\n",
    "y_pred = per_clf.predict([[2, 0.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure perceptron_iris_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAEYCAYAAABBfQDEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XmcTvX7x/HXNYshjH2pbCEp+dpLqWhTSqW0fi1ZIqLlV5GiFGOfQWRJRUWWiCJli2Q3o81Xi6SoZJd9G/P5/XGPuzFmM2buM8v7+XicR/d9nc8557pnmC5nPudzmXMOERERERHxCfI6ARERERGRrEQFsoiIiIhIAiqQRUREREQSUIEsIiIiIpKACmQRERERkQRUIIuIiIiIJKACWUREREQkgYAVyGYWZmbvmNkWMztoZt+aWZMUxv+fmW03swNmNt7MwhLsq2BmS8zsiJn9ZGa3BOZTiIiIiEhOF8g7yCHAH0BDoBDQC/jQzCokHmhmtwE9gJuB8kBF4LUEQ6YA3wDFgJ7ADDMrkYm5i4iIiEguYV520jOz74HXnHMfJYpPBn53zr0U//5m4APnXGkzqwKsB4o75w7G718Wv39sYD+BiIiIiOQ0IV5d2MxKAVWADUnsrgZ8kuD9d0ApMysWv2/z6eI4wf5qyVynI9ARIH/+vHUuu+ziDMg+8x0/fpLfftvBkSPH/bHg4FCKFbuEvHkLepiZiIiIiDe2bk1+X7lyaTnmd5zbbaldx5MC2cxCgQ+A95xzPyUxpACwP8H7068LJrHv9P4kK1/n3DhgHECdOpXd6tVR55F5YJ04cZJXXvmAoUM/BuDUqZPs2vULd9zxMnfc8TLBwZ79+0ZEREQk4Dp1Sn7fSy+l5Zi6abpOwFexMLMgYCJwAuiazLBDQHiC96dfH0xi3+n9B8lh8uQJZeDANsye/TLFi/s+snOOuXP7MGzYTezd+4fHGYqIiIjkPAEtkM3MgHeAUkBz59zJZIZuAGokeF8D2OGc2xO/r6KZFUy0P6mpGjnC7bfXISZmOI0aVffHNm1aRr9+Nfnuu9keZiYiIiISOOGJb5GmEk9tX3IC+pCemY0FagK3OOcOpTDuduBd4CZgGzATWOuc6xG/fzWwHN9KGE2ACcClzrldKV0/u02xSOzUqVMMGvQRffpMJS4uzh+/8canuO++wYSGhqVwtIiIiEju1qmTrXPOpTrPIpDrIJcHHsdXIG83s0PxWwszKxf/uhyAc24eMBhYAmwFtgC9E5zuYXyTSPYBA4H7UyuOc4Lg4GBeeulBFi3qS5kyxfzxJUtGMHjwNezYsdHD7ERERERyBk+XeQu07H4HOaG9ew/SocNI5sxZ64+FheXnkUfGUL9+Kw8zExEREcmastwdZMlYRYsWZMaMFxk+vAN58vhWszh+/DDvvtuad999lGPHkp3BIiIiIiIpUIGcjZkZTzxxJ8uXD+bSSy/yx1evfp8BA+rwxx/fepidiIiISPakKRY5xKFDR3nqqXFMmrTEHwsJyUPz5lE0atQF3wIiIiIiIrlL9+5w4MDpd3VxLibVokh3kHOIAgXyMX7807zzztPkz58XgNjYE0yb9iRjx97L4cN7Pc5QREREJPD+LY7TTgVyDtOq1Y2sWRNFjRqX+GPfffcJERE12bRpuYeZiYiIiGQPKpBzoCpVLmb58sF07drUH9u37w+iohry2WcRxMWd8jA7ERERkaxNBXIOFRYWytChj/HRRy9RtKiv6aBzccye/TKvv34r//yzzeMMRURERLImFcg53F13XUV09FAaNLjcH/v55yVERNTgf//73MPMRERERLImFci5QNmyJVi4MIKePR/yr2Zx6NBu3njjDmbMeJ7Y2BMeZygiIiKSOcLDz/0YLfOWy3z55XoefXQof/+9zx8rX74ejz02lRIlKnqYmYiIiEjmUic9SVKjRtWJiRlOkyZ1/LEtW6Lp168WMTHTPMxMREREJGtQgZwLlShRiFmzejJ4cFtCQ31tqo8dO8Dbbz/MxIkdOHHiiMcZioiIiHhHBXIuFRQUxDPP3MPSpQOoWLGUP75ixdsMGFCPv/5a72F2IiIiIt7RHGThwIEjPPHEGD78cJk/FhqalwceGM7113dUm2oRERHJcJ06Jb9v7Nik4507Q1KlqxmMGZOWa2oOsqRRePgFTJz4LOPGdSVfvjwAnDx5jMmTO/HWWw9x5Mg/HmcoIiIiknRxnFI8vVQgCwBmRps2t7BqVRRXXlneH//66+n061eLzZtXe5idiIiISOCoQJYzXHFFWVasGMzjj9/uj+3Z8zuRkdczf/5g4uLiPMxOREREJPOpQJaz5MsXxsiRnZgypTuFCl0AQFxcLLNmvcAbbzThwIEdHmcoIiIiknkCWiCbWVczizGz42b2bgrjxprZoQTbcTM7mGD/l2Z2LMH+nwPyAXKZ5s2vJTp6GFdffZk/9sMPC4iIqMGPPy7yMDMRERGRzBPoO8jbgAhgfEqDnHOdnHMFTm/AFGB6omFdE4y5LInTSAaoUKEUixf3o1u35v7YgQM7GDGiMR9//BKnTp30MDsRERHJTZJbWCujF9zyZJk3M4sAyjjn2qRhbH5gO9DUObc0PvYlMMk59/a5XFfLvJ2fhQu/oW3b4ezcud8fq1jxGtq3n0KxYuVTOFJERETEezlpmbfmwC7gq0TxAWa228xWmFmj5A42s47x0zpidu8+kJl55ni33lqLmJjh3HJLDX9s8+ZV9OtXk2++melhZiIiIiIZJzsUyI8C77szb3W/AFQELgbGAXPMrFJSBzvnxjnn6jrn6hYvHp752eZwpUsX4dNPexMR0YrgYN8fnyNH/uHNN5szefITnDhx1OMMRURERM5Pli6Qzawc0Ah4P2HcObfGOXfQOXfcOfcesAK4w4MUc6WgoCC6d2/OkiX9KV++hD/+1VdjGDSoPn///aOH2YmIiIicnxCvE0hFK2CFc25zKuMcoH7IAVa/flWio4fx+OOjmDVrFQB//fU9AwbU5eGH3+Caa9qoTbWIiIjHuneHA0nMMg0Ph8GDA59PoJ35+evUScsxgV7mLcTM8gLBQLCZ5TWzlIr01sC7ic5R2MxuO32smbUAbgDmZVrikqzChQswdWp33nijE2FhoQCcOHGE999vx/jxLTl6VPO+RUREvJRUcZxSPKdJz+cM9BSLXsBRoAfQMv51LzMrF7+ecbnTA83sGqAMZy/vFopvqbhdwG7gSaCZc25jAPKXJJgZHTvezsqVQ6hatYw/Hh09mf79a7NlS4yH2YmIiIicm4AWyM65V51zlmh71Tm3NX49460Jxq5yzuV3zh1MdI5dzrl6zrmCzrnCzrn6zrmFgfwckrTq1SuwalUkbdve4o/t2vUrkZHXsGjRMLxYUlBERETkXGXph/Qk+8mfPy9vvtmV999/loIF8wFw8mQsM2Y8y+jRd3Ho0G6PMxQRERFJmQpkyRQPP3wDa9cOpU6dyv7Y+vVz6du3Bhs3LvUwMxEREZGUqUCWTFOp0oUsXTqA//u/e/yx/fu3MWzYTcyZ8yqnTsV6mJ2IiEjuEJ5MG4jk4jlNej6nJ62mvaJW0975/PMY2rcfQcJuhpdeegPt2n1AkSJlUjhSREREJGPkpFbTkgM0aVKX6OhhNGx4pT/2yy9fERFRg++/n+NhZiIiIiJnUoEsAXPxxcWYN+81evd+hKAg3x+9w4f3Mnr03Xz44TOcPHnc4wxFREREVCBLgAUHB9Oz50MsWtSXMmWK+eOLF7/OkCHXsmPHLx5mJyIiIqI5yOKhPXsO0KHDG3z66Vp/LCysAP/97xiuvrqlh5mJiIjkHIFoNZ1d2llrDrJkecWKhfPRRy8ybNhj5Mnj6zh+/PghJkxoxbvvtuHYsUMeZygiIpL9BaLVdE5rZ60CWTxlZnTp0pRlywZRufJF/vjq1e8xYEBd/vjjWw+zExERkdxIBbJkCbVqVWLNmihatGjkj+3Y8TODBtXnyy9HqU21iIiIBIwKZMkyChbMx4QJz/DOO0+TP39eAGJjjzN1alfGjr2Pw4f3epyhiIiI5AYqkCXLadXqRlavjqJGjUv8se+++5iIiJps2rTCw8xEREQkN1CBLFnSZZddzLJlg+jS5U5/bN++Pxg6tCGffdaPuLhTHmYnIiKSfQSi1XROa2etZd4ky5s9ew0dOoxk375/V7W47LKbaNduEoUKXehhZiIiIpKdaJk3yTHuvvtqoqOH0aDB5f7Yzz8vJiKiBhs2zPMwMxEREcmJVCBLtlCuXAkWLozgxRcfwMwAOHhwFyNHNuGjj7oTG3vC4wxFREQkp1CBLNlGSEgwr73WgnnzXuPCC4v44wsXDiEy8np27drsYXYiIiKSUwR0DrKZdQXaANWBKc65NsmMawO8AxxNEG7qnPsyfn8FYAJwNbAV6OqcW5Ta9TUHOefYufMfHntsBPPmfe2P5c0bTsuWb1G37oMeZiYiIrlRVm613KlT8vvGjj07lp7PEqjP37kzJFW6msGYMWnJrS7OxVhq1wn0HeRtQAQwPg1jVznnCiTYvkywbwrwDVAM6AnMMLMSGZ6tZFklSxbm4497MWhQG0JCggE4duwAb7/9EJMmdeTEiSMeZygiIrlJTmq1nJ7PEqjPn9x93ZTu96Ynh4AWyM65mc65j4E96T2HmVUBagO9nXNHnXMfAeuB5hmUpmQTQUFB/N//NWPp0gFUrFjKH1++/C0GDKjHX3/9z8PsREREJLvKynOQa5nZbjPbaGYvm1lIfLwasNk5dzDB2O/i42cxs45mFmNmMbt3Z8N/xkmq6tWrwpo1Q3nggev8sb///oGBA+uxbNlbalMtIiIi5ySrFshfAVcCJfHdGX4E6Ba/rwCwP9H4/UDBpE7knBvnnKvrnKtbvHg2Xa1aUlWoUH4mTXqON9/sQr58eQA4efIYH3zQkbfffpijRxP/kRERERFJWpYskJ1zm51zvznn4pxz64E+wP3xuw8BiSvdcOAgkquZGW3b3sqqVVFUq1bOH1+37kMiImry229rPMxOREREsossWSAnwQGnnzjcAFQ0s4R3jGvEx0W44oqyrFw5hA4dbvPH9uz5naioBixYMIS4uDgPsxMRkZwoJ7VaTs9nCdTnt2TWn0gunt4cAr3MWwgQAvQGygAdgFjnXGyicU2Ar51zO8ysKjADmO6cey1+/2pgOdALaIJvybdLnXO7Urq+lnnLfWbMWEHnzqPYv//fVS2uuOI22rR5n/Dwkh5mJiIiIoGWVVtN98K3tnEPoGX8615mVs7MDpnZ6d+L3wx8b2aHgc+AmUD/BOd5GKgL7AMGAvenVhxL7nT//Q1Yu3YYV11VxR/74Yf5RETU4KefvvAwMxEREcmqAnoH2Wu6g5x7nTwZS+/ek4mMnOmPmRm33fYid931GsHBISkcLSIiIjlBVr2DLOKJ0NAQ+vdvzdy5vSlZshAAzjnmzetPVFRD9u7d6nGGIiIiklXoDrLkOtu376Nt2+F88cV3/ljhwvl56KGJ1Kp1r4eZiYhISrJyO+dASU+r5XOVnq/z+beATtt1zpfuIIsko3TpIsyd25u+fVsSHOz7K/DPP4d58837mDKlCydPHvM4QxERSUpOauecXulptXyu0vN1zsgW0Fnh+6kCWXKloKAgXnjhfhYv7k+5ciX88aVLRzNw4NVs3/6Th9mJiIiIl1QgS652zTVViY4eRrNm9f2xv/76nv7967By5btqUy0iIpILqUCWXK9IkQJMm/YCI0c+TlhYKAAnThzh/ffbMmFCK44dU5NGERGR3EQFsgi+Jd8ef7wJK1YM4bLLyvjja9d+QL9+tdmyZZ2H2YmIiEggqUAWSeA//6nA6tWRtGlzsz+2a9cmBg++hi++GK4pFyIiHspJ7ZzTKz2tls9Ver7OGdkCOit8P7XMm0gypkxZSpcuYzh06N9VLapXb8qjj06gQIHiHmYmIiIi6aFl3kTO0yOPNGTt2mHUrl3JH1u//lMiImqyceNSDzMTERGRzKQCWSQFlStfyFdfDeTpp+/2x/755y+GDbuJTz99jbi4Ux5mJyIiIplBBbJIKvLkCWXIkHbMmtWTYsUKAuBcHJ9++irDht3Mvn1/epyhiIiIZKQ0F8hmdoGZXWtmzczsvoRbZiYoklXceWc9YmKGc8MN1fyxX35ZSkRETb7//lMPMxMREZGMFJKWQWZ2CzAFKJbEbgcEZ2RSIlnVxRcXY/78PgwYMIOIiGnExcVx+PAeRo++i5tueoZ77x1IaGiY12mKiJyX7t2TbvcbHg6DBwc+n/PRqVPy+8aOTTreuXPSLZLNYMwYb49Jz/fmXI/JSd//9ErrHeTXgblAGedcUKJNxbHkKsHBwfTq9RALFvTh4ov//Tfj4sXDGTKkATt3bvIwOxGR85dUcZRSPKdJboGvlBb+CtQx6fnenOsxuf37D2kvkCsAfZ1z2zIxF5Fs5YYbriQ6ehh33PHvajFbt66jX79arF072cPMRERE5HyktUBeAVyWmYmIZEfFi4cza1ZPoqLaExrqm7F0/Pghxo9vwfvvt+P48cMeZygiIiLnKtk5yGZWO8HbsUCkmV0ErAdOJhzrnPs6c9ITyfrMjCefvIsGDS6nZcsoNm36G4CVKyfw668r6dBhGmXK1PA4SxEREUmrlO4gxwDR8f+dAVQFxgGr4mMxCcakiZl1NbMYMztuZu+mMO5RM1tnZgfM7E8zG2xmIQn2f2lmx8zsUPz2c1pzEMkstWtXZs2aoTzySEN/bMeOnxk48Gq+/HK02lSLiIhkEykVyJcAFeP/m9JW8Ryutw2IAManMu4C4BmgOHA1cDPwfKIxXZ1zBeI3Tf+QLKFgwXy8++4zvP32U1xwgW81i9jY40yd2oU332zO4cP7PM5QRCR14eHnFs9pzM4tHshj0vO9Oddjcvv3H8DSclfLzG4AVjrnYhPFQ4BrnXNfndNFzSLwrYjRJo3jnwVudM7dFf/+S2CSc+7tc7lunTqV3erVUedyiEi6/fTTn7RsGcn33//ujxUtWo727adQqdK13iUmIiKSS3XqZOucc3VTG5fWh/SWAEWTiBeK35fZbgA2JIoNMLPdZrbCzBold6CZdYyf1hGze3cuWp9EPFe1ahmWLx9M5853+GN7924lKuoGPv+8v9pUi4iIZFFpLZANX0OQxIoBmfqYvpm1A+oCkQnCL+Cb2nExvnnRc8ysUlLHO+fGOefqOufqFi+ei343IFlC3rx5eP31jnz4YQ+KFCkAQFzcKT75pCcjRtzG/v3bPc5QREREEkuxQDaz2WY2G19xPOn0+/htLrAQWJlZyZlZM2AA0MQ5t/t03Dm3xjl30Dl33Dn3Hr5l6O5I7jwiXmvWrD7R0cO49trL/bGffvqCiIgabNgw38PMREREJLHUWk3vif+vAfuAown2nQCWA29lQl6Y2e3x577TObc+leEuPkeRLKtcuRIsWhRBnz5TGTRoBs45Dh7cyciRt9O4cXfuuSeC4OBQr9MUEcmyAtFmOZC5ZeXrnKusmld6pVggO+faApjZ70Ckc+68plPEP9QXAgQDwWaWF4hN4uG/m4APgHudc2sT7SuMb2WLpUAs8BC+OcpPn09uIoEQEhJMnz4taNToStq0Gc727b5VLRYsGMwvvyylffspFC9+icdZiohkTYFos5xeOe065yqr5pVeaZqD7Jx77XyL43i98N2F7gG0jH/dy8zKxa9nXC5+3Mv4HgD8LMFax5/H7wvFt1TcLmA38CTQzDm3MQPyEwmIm26qQUzMMBo3ruWP/fbbGiIiarJu3XQPMxMREZGUOun9RtIP5p3FOZemtZCdc68Cryazu0CCcTemcI5dQL20XE8kKytZsjCzZ7/M8OGz6dVrIrGxpzh27ABvvfUgP/3UkQceGEaePBd4naaIiEiuk9Id5DeAUfHbe/hWrPgVmBS//RofezdzUxTJuYKCgnj22WYsXTqASy4p5Y8vWzaOgQOvYtu2xKsbioiISGZLtkB2zkWd3vB1zBvknLvVOfdK/HYrMBCoEqhkRXKqevWqsHbtUJo3/7eByLZtGxgwoB7Ll7+tNtUiIiIBlNZ1kO8DPkwiPh24O+PSEcm9ChXKz+TJ3Rgz5gny5s0DwMmTR5k0qQPvvPMIR4/u9zhDERFvBaLNcnrltOucq6yaV3qltdX038DLiVs7m9ljQIRzrnQm5Zeh1GpasosNG7bSokUkP/yw1R8rXvwS2refyiWXXOVhZiIiItlXRreaHgaMMrOxZtYmfhsLjIzfJyIZqFq1cqxcOYTHHmvsj+3e/RtRUdeyYEEkcXFxHmYnIiKSs6V1mbfBQCugOjA0fqsOPOqcG5R56YnkXhdcEMbo0U/wwQfPEx7uW80iNvYUM2d2Y9SoOzlwYKfHGYqIiORMab2DjHPuQ+dcA+dc0fitgXMuqXnJIpKBHnjgOqKjh1Gv3qX+2IYN8+jXryY//bTYw8xERERyptRaTYtIFnDJJaVYsqQ/r7zyAUOHfgzA/v1/8/rrt3D77S/RtOmrBAfrr7NIUnJaC9ycIiu3jRZJ9g6ymR0ws+Lxrw/Gv09yC1y6IrlXnjyhDBzYhjlzXqFEiUIAOOf4/PN+DB3aiL17t6ZyBpHcKae1wM0psnLbaJGUbjk9CRxM8FoLsYpkAbfdVpuYmGG0bTucxYu/B+DXX1cQEVGT1q3HU7NmM48zFBERyd6SLZCdc+8leP1uQLIRkTS58MKizJ3bmyFDZvLaa1M4dSqOI0f2MXbsvTRq1JXmzYcQGprX6zRFRESypTQ9pGdmL5nZNWamSY4iWURwcDA9ejzAF1/0o2zZ4v74l1++waBB9dm+/WcPsxMREcm+0rqKRRNgCbDPzBbEF8zXqmAW8d61115OdPQw7rmnvj/255/fMWBAHVatei+FI0VERCQpaV0H+XqgCHAvsAZfwfwFvoJ5fualJyJpUbRoQT788AVGjOhIWFgoAMePH+a999owYUIrjh07mMoZRHKunNYCN6fIym2jRdLUavqMA8xKATcBdwIPArHOuQsyIbcMp1bTkht8991vtGgRycaNf/ljJUpUpkOHaZQrV9vDzERERLyVoa2mzexBMxttZj8Cm4EOwC/ArfjuLItIFlGjxiWsWRNF69Y3+WO7dm1i8OBrWLx4BOf6j2IREZHcJq1zkKcCzYHxQAnn3E3Oudecc0udc8czLz0RSY/8+fPy9ttPMWHCMxQo4FvNIjb2BB9++DRjxtzDoUN7PM5QREQk60prgdwRWIBvPeRtZjbHzJ4zs9pmZpmXnoicjxYtGrFmzVBq1aroj33//RwiImrwyy9feZiZiIhI1pWeOciVgEb4plfcCxxyzhVL47FdgTZAdWCKc65NCmP/D3gBuACYAXQ+fbfazCoAE4Crga1AV+fcotSurznIklsdP36Snj3fZ8SIOf6YWRBNm/amSZOeBAUFe5idSO7UuTMk9b9gMxgzJvtdJ6u2gVZLa0koQ+cgA5hZkJldDdyP7+G8poABG88hr21ABL6pGild6zagB3AzUB6oCLyWYMgU4BugGNATmGFmJc4hD5FcJSwslMjI9syc+RLFihUEwLk45szpzbBhN7Nv31+pnEFEMlpy96cy+jGBQF0nq7aBVktrSY+0PqT3ObAPWAY0A77GNye5iHPumrRezDk30zn3MZDaBMhHgXeccxucc/uAvvjuPGNmVYDaQG/n3FHn3EfA+vh8RCQFTZteRXT0MK6/vpo/9ssvS4mIqMH69XM9zExERCTrSOsd5G/x3TUu4py7xjn3onNuvnPucCblVQ34LsH774BSZlYsft9m59zBRPurkQQz62hmMWYWs3u3/uknUqZMcRYs6EOvXg8RFOT7EXD48B5GjWrK9OnPEht7wuMMRUREvJXWRiGZXRAnVgDYn+D96dcFk9h3en/BpE7knBvnnKvrnKtbvLhWEhcBX5vqV155hPnzX+Oii4r64198MYzBg69l585NHmYnIiLirTTPQQ6wQ0DCavb064NJ7Du9X63CRM5Rw4bViYkZzh13/Pu8wtat6+jfvzbR0VM8zExERMQ7WbVA3gDUSPC+BrDDObcnfl9FMyuYaP+GAOYnkmMULx7OrFk9iYxsR2hoCADHjh3knXf+y/vvt+f48UD94kgkd0lukdSMXjw1UNfJqm2g1dJa0uOcl3k7r4uZhQAhQG+gDL6OfLHOudhE424H3sXX0nobMBNY65zrEb9/NbAc6AU0wbfk26XOuV0pXV/LvImk7OuvN9GiRSS//rrdHytduiqPPTaNMmX+42FmIiIi5y/Dl3nLIL2Ao/iWcGsZ/7qXmZUzs0NmVg7AOTcPGAwswbfO8RZ8RfVpDwN18a2sMRC4P7XiWERSV7t2ZdasGcrDD9/gj23f/hMDB17F0qVj1KZaRERyhYDeQfaa7iCLpI1zjokTF/PUU+M4cuTfbvK1ajWnZcu3yJ+/iIfZiYiIpM9530E2s4NmdiAtW8amLiJeMzNat76Z1aujqF69gj/+zTcf0a9fLTZvXuVdciIiIpksJIV9XQOWhYhkSVWrlmH58kG88MK7jB37OQB7924hMvJ67r67L40bv+BfS1lERCSn0BQLEUmTWbNW8fjjb/DPP/+ualG16i20bTuRQoVKe5iZiIhI2mTVh/REJJu6995riI4eRv36l/ljP/20iIiIGvzwwwIPMxMREclYaSqQzSyPmb1mZhvN7JiZnUq4ZXaSIpI1lC9fki++6McLL9yPxS+ievDgTkaMuI1Zs3pw6tRJjzMUERE5f2m9g9wXeBSIAuKAbsAoYA/wROakJiJZUWhoCH37tuSzz16lVKnC/vj8+YOIjLyB3bt/8zA7ERGR85fWAvlBoJNz7k3gFPCJc+4pfGsT35pZyYlI1nXzzTWIiRlO48a1/LHffltNv361WLduhoeZiYiInJ+0FsilgB/iXx8CTt82mgc0zuikRCR7KFWqMLNnv8yAAY8SEhIMwNGj+3nrrQf44INOnDhx1OMMRUREzl1aC+StwEXxrzcBt8W/vgZfNzwRyaWCgoJ47rl7WbKkPxUqlPTHly17k4EDr2Lbth9SOFpERCRkH9FdAAAgAElEQVTrSWuBPAu4Of7168BrZvYb8C7wdibkJSLZzNVXX8batUNp3vxaf2zbtv8xYEBdVqx4R22qRUQk20hTgeyce9E51y/+9QzgOmAkcJ9zrmcm5ici2UjhwgWYPLkbo0d3Jm/ePACcPHmUiRMf4513/svRo/s9zlBERCR1aV3m7QYz83fdc86tcc4NBeaZ2Q2Zlp2IZDtmxmOP3cbKlUO4/PKy/nhMzFT69avN779He5idiIhI6tI6xWIJUDSJeKH4fSIiZ7jyyvKsWhVJ+/b/LnSze/dmBg++loULo4iLi/MwOxERkeSltUA2IKkJhMWAw0nERUS44IIwxozpwqRJzxMefgEAcXGxfPTR84wa1ZSDB3d5nKGIiMjZUiyQzWy2mc3GVxxPOv0+fpsLLARWBiJREcm+HnzwOtauHUrdupf6Yxs2fE5ERA1+/lm/hBIRkawltTvIe+I3A/YleL8H+BMYC7TMzARFJGeoWLE0X37Zn2efbeaP7d//N8OH38zs2a9w6lSsh9mJiIj8KySlnc65tgBm9jsQ6ZzTdAoRSbc8eUIZOLANjRpVp337EezatR/nHJ991peNG5fQrt1kihYtm/qJREREMlFal3l7zTl32MzqmtlDZpYfwMzyJ1zdQkQkLW6/vQ7R0cO48cbq/timTcuJiKjBt99+4mFmIiIiaV/mrZSZrQbWApPxtZ4GGApEpfViZlbUzGaZ2WEz22Jm/01m3OdmdijBdsLM1ifY/7uZHU2wf0FacxCRrOGii4ry2Wev8tprLQgK8v0oOnJkH2PHNmPatKc4efKYxxmKiEhuldZVLIYBO/CtWnEkQXw60PgcrjcKOIGvwG4BjDGzaokHOeeaOOcKnN7wPQg4PdGwuxKMOZccRCSLCA4O5sUXH+CLLyIoW7a4P75kyUgGD76GHTs2epidiIjkVmktkG8Gejrn9iWK/wqUS8sJ4qdlNAdeds4dcs4tB2YDrVI5rgJwPfB+GnMVkWymQYMriI4ext13X+2P/fHHt/TvX5vVq/VXX0REAiutBXI+fHd+EysBpPX3oFWAWOdcwltC3wFn3UFOpDWwzDn3e6L4B2a2y8wWmFmN5A42s45mFmNmMbt3H0hjqiISaEWLFmT69B68/npH8uTxPdpw/Phh3n33USZMaM2xY4c8zlBERHKLtBbIXwFtErx3ZhYMvAB8kcZzFAASV6j7gYKpHNcaeDdRrAVQASiPr5PffDMrnNTBzrlxzrm6zrm6xYuHpzFVEfGCmdG58x0sXz6YSy+9yB9fs2Yi/fvXZuvWbzzMTkREcou0FsjdgQ5mthAIw/dg3g9AA+DFNJ7jEJC4Qg0HDiZ3gJldB5QGZiSMO+dWOOeOOueOOOcGAP/gm4YhIjlAzZoVWbMmilatbvTHdu78hcGD67N48QicS6qxp4iISMZI6zJvPwD/AVYBC4C8+B6aq+Wc+zWN19oIhJjZpQliNYANKRzzKDDTOZfa71YdvmYmIpJDFCiQj3feeZrx458mf/68AMTGnuDDD59mzJhmHDq0x+MMRUQkp0rrHWScc387515xzjV1zt3hnOvlnPv7HI4/DMwE+sSvn9wAuAeYmNR4M8sHPEii6RVmVs7MGphZHjPLa2bdgOLAirTmIiLZR8uWN7JmTRQ1a1b0x77/fjb9+tXkl1+WeZiZiIjkVCkWyGZ2gZmNMrO/zGynmU02s+IpHZOKJ/A98LcTmAJ0ds5tMLPrzSzxXeJm+KZOLEkULwiMwdf6+i/gdqCJc063k0RyqCpVLmbZskE8+WRTf2zfvj8ZOrQRc+f2JS7ulIfZiYhITmMpzeUzsyH4itoP8K1W8QjwpXPugcCkl7Hq1KnsVq9Oc18TEcmC5sxZS4cOI9m799/HF6pUaUS7dh9QuPBFKRwpIiK5XadOts45Vze1calNsbgPaO+c6+icewq4E2gWv4KFiEjA3XXXVcTEDOO6667wxzZu/JKIiBqsX/+Zh5mJiEhOkVqBXBbwT/Jzzq0FYgHdphERz5QpU5wFC/rSs+dDmPmezz10aDejRt3JjBnPExub1LLtIiIiaZNagRzM2Q1CYoGQzElHRCRtQkKC6d37ERYs6MNFFxX1xxctimLIkAbs2pXWBXZERETOlFqBbMAkM5t9esO3xNtbiWIiIp5o2LA60dHDaNKkjj+2ZUsM/frVIjp6qoeZiYhIdpVagfwesA3Yk2CbBPyRKCYi4pkSJQoxa1ZPhgxpR2io7xdcx44d5J13HmHixMc4fvywxxmKiEh2kuIqFjmNVrEQyfnWrdtEy5aR/Prrdn+sdOnL6dBhGhdfXN3DzERExGsZtYqFiEi2UqdOZdasGcpDD/3bfX779h8ZOPAqvvpqrNpUi4hIqlQgi0iSdu5cSkxMB1asuJeYmA7s3LnU65TSLDz8At5//1nGjetKvnx5ADh58hiTJ3dm3LgHOHLkH48zFBGRrEwFsoicZefOpfz662iOH98FOI4f38Wvv47OVkWymdGmzS2sXh3FlVeW98e/+eYjIiJqsnnzag+zExGRrEwFsoicZevWScTFHT8jFhd3nK1bJ3mUUfpdfnlZVqwYzOOP3+6P7d27haioBsyfP4i4uDgPsxMRkaxIBbKInOX48d3nFM/q8uULY+TITkyd2p1ChS4A4NSpOGbN6sHIkbdz4MAOjzMUEZGsRAWyiJwlLKz4OcWzi/vuu5bo6GHUr3+ZP/bjjwuJiKjBDz8s9DAzERHJSlQgi8hZypVrSVBQ2BmxoKAwypVr6VFGGadChVJ88UU/undv7m9TfeDADkaOvI1Zs17k1KmTHmcoIiJeU4EsImcpWbIhlSo9QVhYCcAICytBpUpPULJkQ69TyxChoSFERLRi7tzelCxZCADnHPPnDyQy8gZ27/7d2wRFRMRTahQiIrna9u37aNduOIsWfeeP5ctXiFat3qF27eYeZiYiIhlNjUJERNKgdOkifPppb/r1a01wsO9H4tGj+xk37n4mT+7MiRNHPc5QREQCTQWyiOR6QUFBdOt2H0uW9Kd8+RL++FdfjWXQoKv5++8fPcxOREQCTQWyiEi8+vWrEh09jHvvvcYf++uv9QwYUJcVK8arTbWISC4R0ALZzIqa2SwzO2xmW8zsv8mMe9XMTprZoQRbxQT7a5rZOjM7Ev/fmoH7FCKSlOzcmjqhwoULMHVqd0aN6kzevL421SdOHGHixPaMH9+Co0cPeJyhiIhktkDfQR4FnABKAS2AMWZWLZmx05xzBRJsmwHMLA/wCTAJKAK8B3wSHxcRD+SE1tQJmRkdOtzGihWDqVq1jD8eHT2F/v1rs2VLjIfZiYhIZgtYgWxm+YHmwMvOuUPOueXAbKDVOZ6qERACDHfOHXfOjQAMuCkj8xWRtMtJrakTql69AqtWRdK27S3+2K5dvzJ48LUsWjRUbapFRHKoQN5BrgLEOuc2Joh9ByR3B/kuM9trZhvMrHOCeDXge3fmZMDvkzuPmXU0sxgzi9m9W78aFckMOa01dUL58+flzTe7MnHicxQsmA+AU6dOMmPGc4wefRcHD+7yOEMREclogSyQCwCJK9T9QMEkxn4IXA6UADoAr5jZIwnOsz+N58E5N845V9c5V7d48fD05i4iKciprakTeuih61m7dih16lT2x/73v8+IiKjJzz9/6V1iIiKS4QJZIB8CEleo4cDBxAOdcz8457Y5504551YCrwP3n+t5RCQwcnJr6oQqVbqQpUsH8H//d48/tn//NoYPv4k5c3pz6lSsh9mJiEhGCQngtTYCIWZ2qXPul/hYDWBDGo51+OYZEz/+OTOzBNMs/oPvAUAR8cDpFtRbt07i+PHdhIUVp1y5ljmmNXVCefKEMmhQWxo1qk779iPYvfsAzjnmzu3Dzz8voV27DyhatKzXaYpkWWZxlCy5m1Kl/iE4+JTX6UgOcupUMDt2FGbnzuI4d373gAPaatrMpuIrdh8DagKfAdc65zYkGncP8BXwD1APmAW85Jx7L361il+AocBYfFMwugGXOudOpHR9tZoWkYy0bdte2rQZxpdfrvfH8ucvSuvWE6hR424PMxPJuipV2sqFFxpFi5YiODgUM0v9IJFUOOc4deoke/fu4O+/Hb/+Wi7JcVm11fQTQD5gJzAF6Oyc22Bm15vZoQTjHgY24Zs28T4wyDn3HkB8EdwMaI2vgG4HNEutOBYRyWgXXVSUzz9/lVdf/S9BQb4fp4cP72XMmHuYNu1pTp48nsoZRHKf8PDDlChxMSEheVQcS4YxM0JC8lCixMWEhx8+7/MFcooFzrm9+IrbxPFl+B6+O/3+kcRjEo3/BqiT4QmKiJyj4OBgXnrpQW64oRqtWw/lzz/3ALBkyQg2bVrGY49NpVSpKh5nKZK1mKmRr2SOjPqzpT+hIiIZ4LrrqhEdPYymTa/yx/744xv696/N6tUTPcxMRETOVUDvIIvImXbuXBqQB9vWr3+FAwe+978PD/8P1av3ydDcAvVZAnWd9ChWLJyPPnqR0aPn8sIL73LiRCzHjx/m3Xdb89NPi3j44VHkzVsg9ROJiIindAdZxCOBas+cuDgGOHDge9avfyXDcgvUZ8kOLa3NjC5dmrJs2WAqV77IH1+9+n0GDKjDH39862F2IpITNWvWiB49unqdRo6iAlnEI4Fqz5y4OE4tnp7cAvVZslNL61q1KrJmTRQtW97oj+3YsZFBg65myZI3COQKQiJy/p58sg0lSxpRUX3PiK9Y8SUlSxp79qS9c2haC9onn2xDixZNUx03YcJMevUakObrJ3bkyBH69XuJq66qTNmyealatTh33tmAmTOnpPkcW7f+TsmSxrffxqQ7j6xEBbKIR7Jye+ZzzS1QnyUrf82SUrBgPsaPf5p33nma/PnzAhAbe4Jp055k7Nh7OXx4r8cZimQ/1apByZJnb9WqZf618+bNy6hRQ9i9O2u0mD9xwreAV5EiRSlQIMmGwmnSrVsnPv54GhERw1mx4iemT1/I/fe3ZN++3PszSgWyiEeycnvmc80tUJ8lK3/NUtKq1Y2sWRNFjRqX+GPfffcJERE12bRpuYeZiWQ/u5KpTZOLZ6QGDW6kbNkKDB3aN8Vxq1Z9xe23X03Zsnm54opSvPzy//mL2SefbMPKlUsZP34UJUsaJUsaW7f+nqbrn76jPGLEIGrUKEPNmmWAs+9If/rpTBo2/A/lyuWjSpWi3HNPQ3bu3JHseefPn83TT79I48ZNKVeuAtWr16Jt2860b9/FP8Y5x8iRg6lXrxLlyuWjYcPqTJ/+72/v6tb1/Xxr3LgeJUsazZo1AiAuLo6oqL7UrFmWMmXCaNiwOp9//skZ14+M7EPt2uUpUyaMatVK06VLa/++xYvncddd13PppUWoUqUoDz54Gxs3/pimr9f5UIEs4pFAtWcOD//POcXTk1ugPkt2bmldpcrFLFs2iC5d7vTH9u37g6iohnz2WQRxceooJpLVBQUF8fLLA3nvvbH89tuvSY75+++/eOSRJlx5ZS2++OIbhg9/h5kzpxAR8SIA/fq9Tt261/DII21Zv/5v1q//m4svTnv3zZUrl/LDD98zdeo8Zsz44qz9O3Zs5/HHH+ahhx5l+fIf+eSTr3jggVYpnrNkydIsXjyPAwf2JztmwIBeTJ78DoMGjWLZsh946qkX6dbtcRYunAvA/PlrAZg6dR7r1//NhAkzARg37nVGjRrCyy8PYunS9TRpci9t297H+vW+5zHmzPmI0aMjGTRoNKtX/8IHH3xK7dr/rgZ0+PBhOnZ8hvnz1zJr1peEhxeiZcu7/P/gyCxaxULEI4Fqz1y9ep9zXsXiXHML1GfJ7i2t8+bNw7BhHbjxxv/QocNI9u07hHNxzJ79Mj//vJi2bSdRuPBFqZ9IRDxzyy13cNVVDRgwoCfjxk09a/+ECaMpVeoiBg8eTVBQEFWqXM7LLw/k+ecfp0ePvoSHFyJPnjzky3cBpUqVPufr582bl9dfH09YWFiS+3fs2MbJkye56677KVu2PACXX35liueMihpH584tqFq1OJdfXp169a7l9tvvoVGjWwFfkTp27FA+/HAB9etfD0D58pfwzTdrGT9+FLfeeifFipUAoGjRYmd8rtGjI3niiedp3vy/APTo0YfVq79i9OhIxoyZxJ9/bqFUqQtp1KgxoaGhlClTjpo1/210d9ddzc/I9fXXJ1CpUjhff72W+vWvO5cv3TlRgSzioZIlGwakuEttSbeknGtugfosgbpOZrr77qupVasirVsPZcUK368Kf/55Cf361eTRR9/jyiubeJyhiKTk5ZcHcccd19ClS7ez9m3c+CN16tT3d9cEuOqq6zhx4gS//baJatWS/+1dWlStemWyxTFAtWo1uOGGW7jhhitp1KgxN9xwC3fddT/Fi5fgzz+3ct11V/jHPvPMSzzzzEtcc80NREdvZt261axdu4Jlyxbz4IONadWqI1FRb7Jx4w8cO3aMhx++Hfi3+2Fs7EnKlq2QbC4HDx5g+/ZtXHVVgzPiV199HYsWfQbA3Xc/wFtvvU7dupdw4423cdNNt3PbbXf7P+Nvv/3KoEEvs27dGvbs2UVcXBxxcXH89dfWdHz10k5TLEREPFC2bAkWLozgpZce9LfbPXhwF2+8cQcffdSN2NjM/fWhiKRf7dpX0bRpc/r06X5Ox2VEa+0LLsif4v7g4GCmT1/Ahx8u4Ior/sPkye9Qv/6l/O9/31G69EUsXvytf3v00U7+40JDQ6lf/3qeeqoH06cvoEePvkycOI6tW38nLi4OgIkT55xx/FdfbeDDDxek63Oc/lpcfHFZVq78mcjINylYMJzevZ/j1lvrcPiwr110y5ZN2b17F5GRbzJv3hoWL/6GkJAQTp7M3J+RKpBFRDwSEhLMq6/+l/nz+3DhhUX88YULIxky5Dp27drsYXYiWVOJEucWzywvvdSf1auXsXjxvDPiVapczrp1q/1FJcDatcvJkycPFSpUAiA0NA+nTmXecwdmRr1619CtW28WLIimdOmL+OSTaYSEhFCxYmX/VqRI0WTPUaWK707z4cOHuOyyKwgLC+PPP7eccXzFipX90zjy5MkDcMbnKlgwnNKlL2Lt2hVnnHvNmuX+84Nv2sitt95J377DmD8/mp9+2sDatSvYu3cPv/zyE8888xING95ClSqXc+jQQWJjYzPsa5UcTbEQEfFYo0bViYkZTvv2rzNv3tcAbNkSTb9+tWjZchx16z7kcYYiWceGDV5n4FOxYmVaterIW2+9fka8bdsnGDduON27P0HHjk+zZctm+vbtQbt2XbngggsAKFeuAt98s5atW38nf/4CFClS9IwpGecjJmY1X321iBtvvI0SJUqxfv03/PXXH2cUpIk1a9aIe+99hJo161KkSDE2bvyB/v1f4tJLq1KlyuUEBwfzxBPP8+qrz+Oco379Gzh8+BDr1q0mKCiI1q07Urx4SfLly8eSJfMpW7YCefPmJTy8EF26dGPQoFeoWPFSatSow/Tpk1i9ehmLFvl+1k2d+i6xsbHUrn01+fMX4JNPphEaGkrFipdSuHARihUrzqRJb3HRRWXZvv0vXnutGyEhmV++qkAW8dCmTWPZsWMBEAcEUapUYypX7pTiMYFoG50eWbkFdHZQokQhPv64FyNGzKFnz4mcPBnLsWMHePvth/nxx0U89NDr5MlzgddpikgCzz33CtOmvXdG7MILL2bKlM957bVu3HRTTcLDC9O8+X/p2bO/f8wTTzxP166Pcv31V3D06FFiYn6jXLkKGZJTeHgh1q5dwdtvj+TAgX+46KKyPPvsyzzwQPKr/dx4421Mnz6RAQN6cvjwIUqWLE3Dhrfy3HOvEBwcDECPHn0pUaIUo0dH0r17ZwoWDKdatZp07eqbZhISEkK/fiOIiupDZORr1K9/PR9//CUdOjzFoUMH6dOnO7t27aBy5csYP/4jrryyRny+hRk5chCvvvo8sbEnqVLlCiZMmEn58r5l48aNm0bPnk/RsOGVXHJJZV59NYp27Zon/UEykOWmbk516lR2q1dHeZ2GCHC6OJ53VrxUqduTLZKTahsNKRfJp9szJ+xAFxQURqVKT2RYARuIa+QmMTG/0LJlJJs3/7tu6YUXXsFjj03j4otTfhpdJKurVetHLrnkcq/TkBzst99+5Jtvkv4z1qmTrXPO1U1yZwKagyziEd+d47THITBto9MjO7WAzg7q1r2UtWuH8eCD1/tjf//9AwMH1uOrr95Um2oRkUymAlnEM3HnGE+fQLRnzm4toLOD8PALmDjxWd58swv58vkefjl58hiTJ3firbce4siRfzzOUEQk51KBLOKZ5P76Zexfy0C0Z86uLaCzOjOjbdtbWbUqimrVyvnjX389nX79arF582oPsxMRyblUIIt4pFSpxucUh8C0jU6P7NwCOju44oqyrFw5hI4db/fH9uz5naFDr2P+/MFnLCclIiLnL6AFspkVNbNZZnbYzLaY2X+TGdfNzP5nZgfN7Dcz65Zo/+9mdtTMDsVv6VulWsRDlSt3olSp2/n3r2FQig/oga8jXuJiOC1toytVeoKwsBKAERZWIsMfngvENXK7fPnCeOONTkyZ0p1ChXyrWcTGnmLWrBd4440mHDiwI5UziIhIWgV0FQszm4KvGmgP1ATmAtc65zYkGtcdWAR8D1QCFgAvOOemxu//HXjMObfoXK6vVSxEJCf4/fcdtGo1lDVrfvbHwsNL0bbtJC6//BYPMxNJnVaxkMyWrVaxMLP8QHPgZefcIefccmA20CrxWOfcYOfc1865WOfcz8AnQIPE40REcqMKFUqxeHE/nn/+Pn/swIEdjBjRmI8/folTp056mJ2ISPYXyCkWVYBY59zGBLHvgGopHWS+Zt3XA4l753xgZrvMbIGZ1Ujh+I5mFmNmMbt3H0hv7iIiWUpoaAj9+7dm7tzelCxZCADnHPPmDSAqqiF79mzxOEMRkewrkAVyASBxhbofKJjKca/iy3NCglgLoAJQHlgCzDezwkkd7Jwb55yr65yrW7x4eDrSFhHJum69tRYxMcO5+eZ/7xNs3ryKfv1q8s03Mz3MTEQk+wpkq+lDQOIKNRw4mNwBZtYVaA1c75zzdyFwzq1IMGyAmT2K7y7znIxLV3KKQLVATk/b6HXrnuTYsT/87/PmLUudOiNTPGbFiubAqQSRYBo0+CjFY1aubIFzh/3vzfJz7bUfpHjMmjXtiI3d638fElKUq68en+z4QH2d1dL6bKVLF2Hu3N5ERs6id+8POHUqjiNH/uHNN5vTsOET3H9/FKGheb1OUyTHa9asEVWrXsnAgW94nYqcp0DeQd4IhJjZpQliNTh76gQAZtYO6AHc7Jz7M5VzO8AyJEvJUU63QD5+fBfgOH58F7/+OpqdO5dm6HX+bRt9ermtOHbsmMemTWOTPSZxcQxw7NgfrFv3ZLLHnF0cA5yKjyctcXEM4NxhVq5skewxiYtjgNjYvaxZ0y7J8YH6OgfqOtlRUFAQ3bs3Z8mS/pQvX8IfX7p0NAMHXs3ff//oYXYi2d+TT7ahRYumKY6ZMGEmvXoNSPc1jhw5Qr9+L3HVVZUpWzYvVasW5847GzBz5pQ0n2Pr1t8pWdL49tuYdOchASyQne//0DOBPmaW38waAPcAExOPNbMWQH/gVufc5kT7yplZAzPLY2Z545eAKw6sSHwekUC1QE5P2+jExXFqcZ/ExXFqcc4qjlOLA2cVx6nFA/V1Vkvr1NWvX5W1a4fRrFl9f+yvv75nwIC6rFw5QW2qJUf4558P2LixAhs2BLFxYwX++Sfl34hlthMnTgBQpEhRChRIbeZo8rp168THH08jImI4K1b8xPTpC7n//pbs25f0z17JPIFuFPIEkA/YCUwBOjvnNpjZ9WZ2KMG4CKAYEJ1grePTt+IKAmOAfcBfwO1AE+fcnoB9Csk2AtcCOTBto7OqQH2d1dI6bYoUKcC0aS8wcuTjhIWFAnDixBHef78d48e35OhRPbAs2dc//3zAtm0dOXlyC+A4eXIL27Z1DGiRfPpu8ogRg6hRoww1a5YBfFMsevTo6h/36aczadjwP5Qrl48qVYpyzz0N2bkz+TXL58+fzdNPv0jjxk0pV64C1avXom3bzrRv38U/xjnHyJGDqVevEuXK5aNhw+pMn/7vTYK6dS8BoHHjepQsaTRr1giAuLg4oqL6UrNmWcqUCaNhw+p8/vknZ1w/MrIPtWuXp0yZMKpVK02XLq39+xYvnsddd13PpZcWoUqVojz44G1s3JhzfzMVyDnIOOf2As2SiC/D9xDf6feXpHCODUDybcNEEggLKx7/6/iz4xkriKSL4dzRrDJQX+fAfT+zPzPj8cebcM01l9OiRSQ//+ybqRYdPZnff1/DY49NpXz5VJcCFclydu7siXNHzog5d4SdO3tSuHDyU8cy2sqVSylYsBBTp85L8jczO3Zs5/HHH6ZnzwE0bdqcw4cPsW5dyu3hS5YszeLF87j77gcIDy+U5JgBA3oxZ84MBg0aRaVKlxETs4rnnutA4cJFuPXWO5k/fy233XYVU6fOo1q1GuTJkweAceNeZ9SoIQwZMpaaNesyffok2ra9j4UL11G9ek3mzPmI0aMjefPNKVx+eXV27955Rr6HDx+mY8dnqFbtPxw9epRhwyL+v727j7Oxzv84/vrMjXE7iMZdBuMmdz/3Zao1bqrVVqifrSyySmtLKltZU1sroWgnsv1SPwkVrfJLKW1tSZL24WbSVkuaUppFRbUbMxiD7++P6zKOY8bccS7mvJ+Px/XA9/pe1/lc51zO+Zzv+d4wdGg/Vq3aWPAYFUl0fHpL1IrUEshlWTa6cuXGpSr3xJay3BuQV5py8AbklaY8Us+zlrQuvQ4dmrJ6dQbDh19YULZz52Yeeuh83n77Ea1o6ZMAABIjSURBVHW5kNNOfn52qcpPlsqVKzNjxhzatGlP27b/dcz+777bTn5+Pv36/ZLk5Ka0adOeoUNvICmpXpHnfPjhWaxfv4bWrety4YVdSE8fzYoVbxXsz83N5YknpjF9+mz69LmEJk2aMXDgYIYO/Q1z5jwGQJ063hiEM86oQ7169ald23vfnjkzg1Gj7mTgwME0b96K9PT7SU3twcyZGQBs3fo19eo1oFevn3PWWcl06tSNESOOtIb36zeQfv0GkpLSknbtOjBjxlyys79i/fq15X8yT0FKkKVCi9QSyGVZNrpr10ePSYaLm8XCm60iPBk+/iwW55+/4JhkuLhZLLp3n3NMMny8WSwi9TxrSeuyqVatMrNm3cIzz9xOjRpVADh4MJ9Fi37HzJn9yclRFxU5fcTHJ5eq/GRp3bo9CQkJRe5v164jaWkXkZbWnuuuG8jcuY/z/ffeL2Bbt2bTtGn1gu2RRx4A4Lzz0li37ksWL17OgAFXs3lzFldf/XPuuOO3AGRlbWTfvn0MGnTJUcfPm/c4W7ZsLjKW3bt38e232zn33KPXXOve/WdkZW0EoH//q8jL20e3bs0YM2YEr7yyiLy8I2M+vvpqMzfeOJhzzmlOSkoi7drV49ChQ2zbFtkvJpES0S4WIkFISuoZkQSqRYsbi53WLVxxU7oVprgp3QpT3JRuhTnelG6FidTzHKnHqYgGDUqjW7eWDB2awfr13ofpJ58sZeLEjowY8RytWul5lVNfUtJktm8feVQ3C7OqJCVNjmgcVasW/SscQGxsLIsWvUlm5mpWrHiT5557ismT7+Lll9+ldet2LF/+j4K6h1t5AeLj40lN7UFqag9uvTWdadMmMWXKvdx2210cOuR15Xv22Vdp1OjoLwTx8fFlug5vPTZo1Kgxf//7Z7z33tusXLmM8ePvICNjAq+/voZq1aoxdOjlNGhwFhkZ/0uDBo2Ii4vjZz9rS37+/jI97qlOLcgiIlGkRYsGrFw5hTFj+heU/fTTdqZP78Orr97HwYMHAoxOpHi1ag2hYcNZxMc3AYz4+CY0bDgrov2PS8rMOOec8xg7djxvvrmO+vUbsmTJ88TFxZGS0qJgC02Qw7Vq1RaA3Nwczj67LQkJCWzd+vVRx6ektKBx4yYABf2BDx48MrtRjRqJ1K/fkLVrj57wa82aVQXnB6/byMUXX8bEidP529/WsWnTBtaufZ8ff/yBzz/fxJgxd9Oz50W0atWGnJzdHDhQcd8v1IIsIhJlKlWK56GHrqdXrw6MGDGDH37YjXOHeO21CWRlvcP11y+gdu2zgg5TpEi1ag05JRPiUJmZq1m5chm9e/flzDPr8cknH7Jt27+OSkjDXXFFL6688ld06tSN2rXrkJW1kQceuJuWLVvTqlUbYmNjGTXqTu67706cc6SmphUM/ouJiWHYsJHUrZtElSpVeOedv9G4cVMqV65MYmJNbr55LFOn/pGUlJZ07NiVRYvms3r1eyxbth6AhQvnceDAAbp06U61atVZsuR54uPjSUlpSa1atalTpy7z5z9Jw4aN+fbbbUyYMJa4uIqbRqoFWUQkSl16aTcyMx+hZ8/2BWWff76SSZM68vHHWphUpDwSE2uydu37DBlyOampLRk//g5uv/1errqq6EHFvXv3ZdGiZ7nmmr5ccEFrxo0bRWpqD1544U1iY73xJ+npExk79j5mzswgLa0dV199MUuXvkhysjcBWFxcHJMn/5kFC2bToUNDhg0bAMBvfnMrN988lvvv/z1pae15/fWXmDPnRdq37+jHW4sFC56if/8e9OzZnqVLX2Tu3MU0adKMmJgYZs16no0bP6Znz/akp9/MuHETqVSp6D7YpzuLphHMXbu2cKtXPxx0GCIip5SDBw8yZcr/MXHi8wV9HAH69LmNK6+cSnx8xf0QlMjr3PlTmjVrE3QYUoF99dWnfPhh4ffYjTfaB865Yue4rLht4yLlsGPHu2Rnzycv73sSEuqSnDz0lBkYVpbYwpe1Lm62DIkusbGx/OEP15CW1p5hw6axbZu37tLy5TP44ov3GDFiIfXqtQw4ShGRyFEXC5EwO3a8y+bNM/0FKRx5eTvZvHkmO3a8G3RoZYotPDkGbznrDz645SRHK6ebHj3akZk5ncsuO6egLDt7PQ880IU1a7Sct4hEDyXIImGys+dz6FDeUWWHDuWRnR18glCW2MKT4+LKJbrVqZPI4sV3M23aDVSq5P3ImJeXw9y51zJv3nD27csJOEIRkZNPCbJImLy8whdNKKo8kk7l2KTiMDNGj76c996bSosWDQvKV69+mgcf7MbWrR8FGJ2IyMmnBFkkTEJC3VKVR9KpHJtUPJ07N2fNmocZPPhIH/fvvvuMKVO6s2LFY1qmWspM946cLCfq3lKCLBImOXkoMTFHj9qPiUkgObnoqXkipSyxhS9nXVy5SKgaNaowb97vmD37VqpW9e69AwfyWLhwNE888d/k5v4YcIRyusnPjyc/f2/QYUgFlZ+/l/z8sq0qGEoJskiYpKSeNG8+ioSEMwEjIeFMmjcfdUrMYlGW2Lp2ffSYZFizWEhpDRvWhzVrptGhQ9OCso8+eplJkzrxxRfvF32gSJjs7CS++WYb+/fvUUuynDDOOfbv38M332wjOzup3OfTPMgiIlJi+/btJz19HjNn/rWgLCYmlssvn8All6QTExMbYHRyukhM3EVy8g7i4/ODDkUqkPz8eLKzk9i1K7HIOiWdB1kJsoiIlNqSJasZOfJ/+Pe/j8xq0br1hVx33bPUrNkgwMhERIpW0gRZXSxERKTUBgxIZd266Zx//pHVqjZteptJkzqyYcMbAUYmIlJ+SpBFRKRMkpPPZNmySdx111WYGQC7d+/k0Ud/wYsv/p4DB/YHHKGISNlENEE2szPM7CUzyzWzr81scBH1zMymmtkP/jbVDr/7evs7mdkHZrbH/7NT5K5CREQOi4uLZcKEIbzxxgTq169dUP7WW38iI6MHO3d+GWB0IiJlE+kW5MeA/UA9YAjwuJm1K6TeSOAKoCPQAegH/BbAzCoBS4D5QG3gaWCJXy4iIgHo3bsDmZnT6du3S0HZli1rmTy5M5mZLwQYmYhI6UUsQTazasBA4F7nXI5zbhXwCnBtIdV/DTzsnNvqnNsGPAwM9/f1AuKAR5xzec65PwMG9DnJlyAiIseRlFSLJUvuYcqU4cTFebNZ7Nu3i9mzr2H+/JHs378n4AhFREomLoKP1Qo44JzLCin7CChsAtd2/r7Qeu1C9n3sjp5+42O//JiRIWY2Eq9FGiCvUqUr/lm28KUCqAtoTebopnsgIKtWPcmqVU8GHQboHoh2ev3l7JJUimSCXB3YFVb2E1CjiLo/hdWr7vdDDt93vPPgnJsFzAIws8ySTO0hFZNef9E9ILoHoptefzGzzJLUi2Qf5BwgfObmRGB3CeomAjl+q3FpziMiIiIiUiqRTJCzgDgzaxlS1hHYUEjdDf6+wuptADqEzmqBN5CvsPOIiIiIiJRKxBJk51wusBi438yqmdkFwADg2UKqPwPcbmaNzKwhcAcwz9+3AjgI3GpmCWY22i9fXoIwZpXjEuT0p9dfdA+I7oHoptdfSnQPRHSpaTM7A5gDXAz8AKQ7554zsx7A68656n49A6YCN/iHzgbGHR6YZ2ad/bK2wKfACOfchxG7EBERERGpsCKaIIuIiIiInOq01LSIiIiISAglyCIiIiIiIaIiQTazM8zsJTPLNbOvzWxw0DFJ5JjZaDPLNLM8M5sXdDwSWf5g3qf8//u7zewfZvaLoOOSyDKz+Wb2jZntMrMsM7uh+KOkojGzlma2z8zmBx2LRJaZrfBf+xx/++x49aMiQQYeA/YD9YAhwONm1u74h0gFsh2YhDdAVKJPHPAvvFU7awL3AC+YWdMAY5LIexBo6pxLBPoDk8ysa8AxSeQ9BqwLOggJzGjnXHV/O+6KehU+QTazasBA4F7nXI5zbhXwCnBtsJFJpDjnFjvnXsabOUWijHMu1zl3n3Nui3PukHNuKfAVoOQoijjnNjjn8g7/09+aBxiSRJiZDQL+A7wddCxy6qvwCTLQCjjgnMsKKfsIUAuySBQys3p47wtaXCjKmNlMM9sDbAK+Af4acEgSIWaWCNwP3B50LBKoB83sezN738x6Ha9iNCTI1YFdYWU/ATUCiEVEAmRm8cAC4Gnn3Kag45HIcs6Nwnvv74G3cFXe8Y+QCmQi8JRzbmvQgUhgxgEpQCO8xUJeNbMif0WKhgQ5B0gMK0sEdgcQi4gExMxi8Fbu3A+MLqa6VFDOuYN+V7uzgJuCjkdOPjPrBFwETA86FgmOc26Nc263cy7POfc08D5waVH14yIXWmCygDgza+mc+9wv64h+XhWJGv7qnE/hDdS91DmXH3BIErw41Ac5WvQCmgLZ3lsB1YFYM2vrnOsSYFwSLAdYUTsrfAuycy4X76e0+82smpldAAzAa0mSKGBmcWZWGYjFe1OsbGbR8OVQjngcaAP0c87tDToYiSwzSzKzQWZW3cxizawv8Cs0WCtazML7MtTJ354AXgP6BhmURI6Z1TKzvoc//81sCJAGvFHUMRU+QfaNAqoAO4C/ADc559SCHD3uAfYC6cBQ/+/3BBqRRIyZNQF+i/fB+G3IHJhDAg5NIsfhdafYCvwbyADGOOdeCTQqiQjn3B7n3LeHN7yul/ucczuDjk0iJh5vutedwPfALcAVYRM4HMWccxGKTURERETk1BctLcgiIiIiIiWiBFlEREREJIQSZBERERGREEqQRURERERCKEEWEREREQmhBFlEREREJIQSZBGR05yZDTeznGLqbDGzOyMV0/GYWVMzc2bWLehYREQKowRZROQEMLN5ftLnzCzfzL40swwzq1bKcyw9mXFGWkW8JhGp+LTcrojIibMMuBZv1aYewGygGt4qbiIicppQC7KIyImT5y9n+y/n3HPAAuCKwzvNrK2ZvWZmu81sh5n9xczq+/vuA34NXBbSEt3L3zfFzD4zs71+V4mHzKxyeQI1s5pmNsuPY7eZvRva5eFwtw0zu9DM/mlmuWb2jpk1CzvPXWb2nV/3GTMbb2ZbirsmXxMze8vM9pjZRjO7uDzXJCJyoihBFhE5efbitSZjZg2AlcA/gXOBi4DqwBIziwEygBfwWqEb+Nvf/fPkAtcDbYBRwCDgD2UNyswMeA1oBFwOdPZjW+7HeVgCcJf/2OcBtYAnQs4zCBjvx9IF+BS4PeT4410TwGTgz0BHYB2w0Myql/W6REROFHWxEBE5CczsXGAw8LZfdBPwkXNuXEidYcCPQDfn3Foz24vfCh16LufcxJB/bjGzB4A7gXvLGF5voBNwpnNur192r5n1w+si8pBfFgfc7Jz7zI83A5hjZuacc8BtwDzn3Gy//oNm1hto5cedU9g1efk5ANOdc6/6ZXcDw/y4VpXxukRETgglyCIiJ84l/mwScXgtx0uAW/x9XYG0ImabaA6sLeqkZvZLYAzQAq/VOdbfyqorUBXYGZKsAlT2Yzks73By7NsOVAJq4yX2rYEnw869Bj9BLoGPw84NkFTCY0VETholyCIiJ85KYCSQD2x3zuWH7IvB69ZQ2FRr3xV1QjNLBRYCE4DfAf8B+uN1XyirGP8xexSyb1fI3w+E7XMhx58IBc+Pc875ybq6/olI4JQgi4icOHucc18UsW89cDXwdVjiHGo/x7YMXwBsC+1mYWZNyhnneqAecMg592U5zrMJOAeYE1J2blidwq5JROSUpm/qIiKR8RhQE3jezLqbWYqZXeTPJFHDr7MFaG9mZ5tZXTOLB7KARmY2xD/mJuBX5YxlGfA+3gDBX5hZMzM7z8wmmFlhrcpFmQEMN7Przaylmf0e6M6RluairklE5JSmBFlEJAKcc9vxWoMPAW8AG/CS5jx/A68/76dAJrATuMAfxPYn4BG8PrsXA38sZywOuBRY7j/mZ3izTZzNkb7AJTnPQmAiMAX4EGiPN8vFvpBqx1xTeWIXEYkE894nRUREys/MXgLinHP9go5FRKSs1AdZRETKxMyq4k1f9wbegL6BwAD/TxGR05ZakEVEpEzMrArwKt5CI1WAz4Gp/iqCIiKnLSXIIiIiIiIhNEhPRERERCSEEmQRERERkRBKkEVEREREQihBFhEREREJoQRZRERERCTE/wPJ+NxdpaYohAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
    "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
    "\n",
    "axes = [0, 5, 0, 2]\n",
    "\n",
    "x0, x1 = np.meshgrid(\n",
    "        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
    "        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
    "    )\n",
    "X_new = np.c_[x0.ravel(), x1.ravel()]\n",
    "y_predict = per_clf.predict(X_new)\n",
    "zz = y_predict.reshape(x0.shape)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Not Iris-Setosa\")\n",
    "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
    "\n",
    "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
    "from matplotlib.colors import ListedColormap\n",
    "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
    "\n",
    "plt.contourf(x0, x1, zz, cmap=custom_cmap)\n",
    "plt.xlabel(\"Petal length\", fontsize=14)\n",
    "plt.ylabel(\"Petal width\", fontsize=14)\n",
    "plt.legend(loc=\"lower right\", fontsize=14)\n",
    "plt.axis(axes)\n",
    "\n",
    "save_fig(\"perceptron_iris_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure activation_functions_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAAEYCAYAAADMNRC5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd8VFX6+PHPM+kdQkggRIz0JkWCoFhQLGABFP2KotgQlbWL9Ycu7q69rGJbWHVhBXEVVKwIKhEQpahUwdC7kNBCSM+c3x9nEpI46VMS8rxfr/uamXvPPffMJdw7zz1NjDEopZRSSimlVHU4/F0ApZRSSimlVMOhAYRSSimllFKq2jSAUEoppZRSSlWbBhBKKaWUUkqpatMAQimllFJKKVVtGkAopZRSSimlqk0DCFVvichWERnng+NMEJE1PjiOQ0Qmich+ETEiMsDbx6yiPFNE5HN/lkEppRoSEblBRLJ8dCwjIlf44lhK1ZToPBDKE0TkFGAZ8JMxpn8N950AXGGM6VZufXPgqDEm20NlTAa2AH2MMctLrY8EQowx+z1xnEqOfwnwETAA2AwcMMbke/OYruMOAOYDzY0xGaXWx2CvAYe8XQallPIFEZkCXO/6WAgcBNYCM4HJxpiCOuYfBkQZY/bVJZ9yeU4B4owxl5Rb3wI4aIzJ89SxlPIUrYFQnjIaeAPoJiKdPZGhMSbdU8FDFcfJ8nbw4NIO2GOMWWyM+cMXwUNljDGHNXhQSh2HvgFaAsnABcBnwBPAQhGJqG2mIhJkjMnxZPBQGdd9QoMHVS9pAKHqzPVE5hpgMvYpz81u0iSKyHRX851sEVkhIueIyA3AX4Gurupa41pXpgmTiLwnIrPK5ekQkR0icp/r8yARWSgiB0XkgIh8XS6Y2eJ6XeY6TqprvzJNmFz5PubKO09EVovI0FLbk137DxeRea7v85uInF/JOZoC/BNo7dp3q2t9qoi8Vj5t6aZFrjRviMhTIpIhIvtE5AURcZRKE+zavs1V5s0icper1mW+K1m669hTKjhOiIi8LCJ7RSRXRH4SkTNKbR/g2n+giCxxfe/lrtqn4jQxIvKuq4y5rnLcU9F5UUopL8hz/fjeZYxZYYx5CVvzewrwIJRcM58VkZ2ua9kyEbmwOINS17uLRGSpiOQDF5ZuwiQiHVxpTi59cBEZ47pWB4lIgIi8LSJbRCRHRDaIyIPF129XDfz1wMWl7oEDXNtKmjCJyGIRebHccaJdeV5eze8UJCITRWS36z6xQ0Se8eiZV42GBhDKE64AthljVgPvAqNEJKh4o9gnPt9jnwYNA04G/uba/D/gReB37BOjlq515U3DXmBjSq0725V+hutzBPAycCr2ZnEY+ExEgl3bT3W9DnLtd3kF3+du4AHgIVdZPwY+EpGe5dI9CUwEemCbb70vtjlURXn+DdjpOnafCtJVZCS2Ov504A7gHuCqUtunAqOA+4DO2CDuELADGO5K09V17LsrOMZzrjxvAnoBq4E5ItKyXLqngYexN+P9wHQREde2f2DP2SVAR1deu2r4XZVSyqOMMWuAORy7Hv4Hew+5BuiGvYZ+JiI9yu36LDAe6AQsKZdnGvbaP7LcPiOBD1zNpRzYa+D/Ya/N/w94FLjRlfYF4AOO1Zq0BBa7+QrTgBGlHxy5vksu8EU1v9NdwGXACKA99nr/u5tjKVU1Y4wuutRpAVKBca73AmzF9mko3n4LcATbxtPd/hOANW7Wby2VbyCwF7i51Pa3gLmVlCsCKALOcH1OBgyQUtnxsRf7x918x2nl8rm11PZWrnVnVFKeccBWN/m+Vm7dFODzcml+LJdmHvCW631717EHVXDcAa7tcRUdx3Wu8oFRpbYHAJuAf5TL58JSafq71iW5Pn8KvOPvv0lddNGlcS7lr5/ltj0DZANtASfQutz2T4A3XO+Lr3fDy6W5Acgq9fkuYBvH+pS2duV9eiVlfAb4pqoyu45/het9M9c1emCp7d9g+3VQze80Efi2uKy66FKXRWsgVJ2ISDvgDOA9AGOMAaZTthlTL2CVKdWBt6aMMYXYmomRruOGYJ++TCtVlrZimzptEpFMbMDhwF7Qq/t9ooFE4IdymxYBXcqtW1Xq/W7Xa3x1j1VDq8p93l3qWL2wN4751F5bIIhS39sYUwT8SM2+95vAVSKy0tXM6uw6lEkppTxJsD/KT3G9/01EsooX4GLstbC05VTufew940zX56uBLcaYkloEEbnN1dwz3XWce6nBfQnA2H56czh2D0wEzuHYPbA632kK0BNIE5HXReTicjUaSlVboL8LoBq80dgn1duPtWJBAETkBGPMDg8eaxrwo4i0AvoCwdhRjYp9jm0idCu2FqEQ+M2VzhPKD1lWMpqHMca4vn9NL8ZOXOerlCA36cqPHGJqcazaqvB7l9rmADDGfCUiJwKDgYHAFyLyoTHmRpRSyr+6YEfAc2CvXX3487U1p9zno5VlaIzZJyLzsD/sF7hepxdvF5GrsE1rx2GbJmUCf8E2JaqpacC/RWQsthnSDmCha1uV38kY84urX9yF2OvzVGCliJxvjHHWojyqEdPIU9WaiARiO389gn2qUbz0wD6lLv7R+CvQXUTiKsgqHxuEVMoYsxTYiH3CMxKYbYwp7szWDNtG9SljzDfGmHVAFGWD5OJRjyo8ljEmE/tUvfxQtGdggxFPS8e2eS2tfBvcqqzA/l8+p4LtVX5vbFOlfEp9bxEJAE6jht/bGJNhjHnXGHMDtibqeleNkVJK+YWIdMP2f5uJvScJ0MIYs7HcUps+W9OAK0WkN7YP2LRS284AlhhjXjPG/GKM2cifazmqdQ/ENhEF28dsJPCeq9af6n4nY8wRY8xMY8zt2NqJc7EjBCpVI1oDoeriYiAO+LcpNwyqiLwP3CYif8c2b3oYmC0iD2NrB7oBR4wx87F9HU50jeaz3bW+oqHrpmNrPZIp2wn6IJAB3CIiO7B9Ep7H1kIU24d9EnOh2FGQco0xh90c43ngbyKyAfgZuBZbPX2Km7R19R3wsogMwXZmuxU4AXtOqsUYkyYiHwBvicjdwC9AEpBsjHkX2z7XYDuhfwbkFAdepfI4KiJvAs+KSAZ2xKp7gQTs8LzVIiJ/cx1/Lfb6cjmwuZJ/T6WU8rQQsXMoOIDm2Kftj2Kv5y+4rnfTgSkicj/2mhWLa44eY8xH7rOt0CfAJOBtYJmxnauLpQE3iMhg7AOwEdiOzgdLpdkKDBaRjtiBKQ4bN/NVGGNyxY5GOB77oOm6UtvSqvpOYkcs3IN96FSA7Wydia25V6pGtAZC1cXNwPzywYPLh9gf+ecbY45iL5g7seNxr8GOyV385GQW8CW2c1c6toahItOwo/scBuYWr3RVv14FdHfl/zrwGJBXKk0htsPbaGwtw+wKjjERG0Q858rrMmxHupWVlKu23im1/IDtbP5xLfIZhQ3UJgLrsW1dYwBcT5/+ih01ai/wmvsseAjbz+Q/2BtMd2zH7D01KEee6zgrsd8nCri0Zl9FKaXq5DzsD+Xt2PvKEOxgGWe57kdga8j/g73Or8c2gT0L+8ClRoydr+hj7I/6aeU2T8KOsvQedsSmZOzIg6X9G1iH7W+Rzp9rwEub5jrOr8aY8rXDVX2nI9gRBpdiA4yewGDjg/mW1PFHZ6JWSimllFJKVZvWQCillFJKKaWqTQMIpZRSXiMid7iGsMwT1yzoFaS7XkR+FpFM10y6z7kGalBKKVXPaAChlFLKm3ZjZyh/p4p04dgZ1uOwwzQPxA59qZRSqp7RpztKKaW8pnhEGxFJwY4OVlG6N0t93OUaUaaioYmVUkr5UYMKIOLi4kxycrK/i8HRo0eJiIjwdzHqDT0fZdX382EKDAUHCgiMCcQR6v1KyPp+PnytvpyPn3/+OcMY09zf5ajEWdjhgN0SkTHAGICwsLDeJ5xwgq/KVSGn04nDoRX7xfR8lKXnoyw9H2XVl/ORlpZWrXtDgwogkpOTWb68qlnlvS81NZUBAwb4uxj1hp6PsvR8lKXno6z6cj5EpMbDVfqKiNwEpGCHXHbLGDMZmAyQkpJi9N5Q/+j5KEvPR1l6PsqqL+ejuveGBhVAKKXqxhiDiPi7GEpVSESGAU8D5xljMvxdHqWUUn/m/7oSpZRPFBwqYGmnpWwevxmd/0XVRyIyCDup1qXGmNX+Lo9SSin3tAZCqUYiY1YGOWk5ZP6YqbUQymdcQ7EGAgFAgIiEAoWumeFLpzsXmA5cZoxZ6vuSKqWUqi6tgVCqkdg7fS8ACSMT/FwS1ciMB3KAh4FrXe/Hi0hrEckSkdaudI8BMcCXrvVZIvKVf4qslFKqMloDoVQjkLszl0Oph5AQofnw+jzwjjreGGMmABMq2BxZKp0O2aqUUg2E1kAo1Qjsm7EPDDS7pBmBMfrcQCmllFK1pwGEUo2ANl9SSimllKdoAKHUce7o2qMcXXmUwCaBNLuomb+Lo5RSSqkGTgMIpY5zxbUPza9sjiNE/8srpZRSqm7014RSxzHjNNp8SSmllFIepQGEUsexwz8cJm97HiEnhBBzZoy/i6OUUkqp44AGEEodx4prH+KvjkccOnmcUkoppepOAwiljlPOfCfpH6QD2nxJKaWUUp6jA8IrdZySAKHz9M4cSj1EZPfIqndQSimllKoGj9ZAiMgdIrJcRPJEZEoVae8VkT9EJFNE3hGREE+WRanGTgKEZoOb0fbZtv4uilJKKaWOI55uwrQb+AfwTmWJRORC4GFgIHAi0AZ4wsNlUUoppZRSSnmYR5swGWM+AhCRFCCpkqTXA28bY9a60v8dmI4NKpRq0CI2boQffgBj/FaGvaviSF8TR9Lpu2mSnOm3cgCcuGULLFrk1zKA/efILwogpyCQ3MJACp0OioyDQqeDwiKxr26WkjROobDo2DpjwCAYA04jJe8NYj9XsD09Yz/rmn2JMYIBjHGld703uNKbsp3ey/81/Wl7uQSGyrcrpZRSteWvPhBdgdmlPq8EEkSkmTFmv5/KpJRHdHruOdiwwa9l+IPnOEgnYtdNpQmf+7UsJ9Vh3xxCSac5+4gnneYcJoZMoitcsogkl1ByCSWHsJL3dgnz2HdSSimlGjN/BRCRwOFSn4vfRwFlAggRGQOMAUhISCA1NdUX5atUVlZWvShHfaHno6w+mfaJ/65hwyiM9E/n5cjsjchWJ7lt49kWcq1fylAsPz+f4ODgP60/kBvJjqzm7M5uxq6sOHYejWN3djPSc2LYnxvN/txojhZ69kd/sKOAkIACgh2FBDoKCXQ4CZAiAsVJgKOIAHES6Hot/b7kVYpwOOw2+3zf4HC9FzEcceST48in0FFEgTgplCIKHIUUSBGRJpCu+U0wziIKA53MjdhBkRRR6CiiUIpwihPEgDgZeLQVJxVGIsAvoen8HGpH00LKViOEGAc3HO5U8nl6zO9kOQpKpTiWvlduHP1y7Whc24OO8MUKj55apZRSjYi/AogsILrU5+L3R8onNMZMBiYDpKSkmAEDBni9cFVJTU2lPpSjvtDzUVau67XVyy/DiSf6tSz1wezZiwgLO4O1a2HdumPL/mrUNQYFQXw8NG9ul6ZNITr6z0tMjH2NiICwMAgNPfZavISEgMMRBARVeDxjDAdyDgDQLLwZAFsObmHqyqnsO7qP9Ox0+3o0nfTsdA7mHGTDnRs4qamtZ7nywyuZ+dtMt3kPSB7AjOvnk5qaSq9+vWjy7IVltjvEQURQBBHBEdx58b8Y2ukiAD5c+yFTV6YSEhhCaGAoIQEhhATY99Eh0TxxzsUleZy56hC5hbklaYMDgkuW5CbJtGnaBoCs/CyidNgKpZRSteSvAGIt0AP4wPW5B7BXmy+p44LTaV8d/plmxRiDiH8mjSsshF9+gSVLji0bN57hNm1UFLRpY2OsE0+E1q3t0qqVDRbi421g4KmvYoyBUv0Cpq6Yym/pv7HzyE52ZtplV+Yu8oryuLvv3bw86GUA9mTt4YnvKx7j4XDescrUlJYpZBdkExMSQ5PQJmWWk5oca8wVFRLFb2N/IyI4oiRoCAkIcfvvdmXXK7my65XV+o4ju4+sVrrIYB3WVymlVO15NIAQkUBXngFAgIiEAoXGmMJySf8LTBGR6diRm8YDUzxZFqX8RYp7q/ohgMj+PZtVg1eReGsirR9q7ZNjbt0KX30Fc+fCd99BZrk+2yEhRaSkBNC9O3TufGxJTPRccFCs0FnI9sPb2XRgE5sObmLjgY1sOriJTQc2sTNzJ+kPpBPgCADgzeVvsmTXkj/lERMSg5QKNNo2bcv4M8cTHxFP84jmxEfE2/fhzWka1pTggGPNsx464yEe4qEqy+kQB52bd/bAN1ZKKaV8z9M1EOOBv5b6fC3whIi8A/wGdDHGbDfGzBGR54D5QBgwq9x+SjVcfqyB2Dt9L7lbcsn+Pdurx9m8GWbOhA8/hOXLy25r3x7694e+fe2yf/8izjvvbI8eP78on7T9aazdt5a2sW1JSUwBYOZvM7l61tUV7rcjcwfJTZIBuLnXzQzpOISk6KSSpVVUKyKCI8rskxCZwN/P/btHy6+UUko1ZJ4exnUCMKGCzWXqzI0xLwEvefL4StUHJTUQAQE+Pa4xhr3T9wKQcG2Cx/PPzYWPPoLJk+H774+tj4iAwYPhwgvh/PP/3O0jNbXu44f+sP0Hftr5Ez/v+ZkVf6wgbX8aRaYIgLtOvaskgGgX246k6CTaNm1rl1j72i62HW1j29IktElJnrf0vqXO5VJKKaUaI3/1gVDquCVF9oetr2sgMn/KJHdzLsGJwTQ5u0nVO1TTH3/AP/8Jb70FB2z/YsLDYdgwuOIKGDTIdlj2hPyifH7e/TM/7fyJu/reVdLc6P6595dpbiQIbZu2pWt8V3q26FmyPiUxhR337vBMYZRSSinllgYQSnman/pAFNc+xF8djwTUvXPBtm3w3HPw9tuQl2fX9eoFt94KV19tRz2qq8y8TBbvWMyi7YtYtH0RS3YtIbfQjmM1qN2gkn4CV3S5gp4tetK7ZW9OaXkKnZt3JjwovO4FUEoppVSNaQChlIdJcR8IHzZhchY4Sf+fnSsgYWTdmi8dOgR//zu8+ioUuKYUGDYMHn7Y9mmoi9IjRK3eu5qek3riNM4yabo070L/E/oTFHBsuNVxp4+r24GVUkop5TEaQCjlaX7oRH1w3kEKMgoI7xxOZM/aDdFZVGRrG8aPh/R0O0LSNdfAI49At261K5cxhu3Z25m4ZCJfb/qakIAQPrrqIwA6xXUiOiSaznGdOaP1GZzR+gxOP+F04sLjancwpZRSSvmEBhBKeZg/hnEt6Tw9MqFWc0Bs2ACjRsFPP9nPZ5wBL78MvXvXvCxFziIWbV/EJ+s/4ZPfP2Hroa0l28ICw8grzCMkMISggCD2jdtXpqZBHX9E5A7gBuBkYIYx5oZK0t4LPASEAzOB240xeT4oplJKqRrQAEIpT/NxDURhViEZn2QAEH9NfI32dTrhjTfgwQchJ8dO4vbSS3DllbWfo+Hfv/yb27+4veRzTFAMF3W8iAvbXsgFbS8gJPDYFMgaPDQKu4F/ABdih+12S0QuBB4GznXt8zHwhGudUkqpekQDCKU8zNfDuGZ8koEz20n06dGEnVT94ZAyMmDkSDsBHMB118HEidCkmgM4OY2TRdsX8d7q92jTtA0P9n8QgEs6XMJLP77EsE7DGNZpGLkbczn3nHNr+rXUccIY8xGAiKQASZUkvR542xiz1pX+78B0NIBQDdi2bXYUu99+g4MHu3P7Cbvpsn4nSbcmknS3/e9waNEh0m5Lq1G+ibf8ef+Y/jF0nNQRgIIDBfx61q81ytPd/kFNg+i1sFdJmpXnryRvT/UrBSvav8fcHiXrNty9gYPfHqxRWXvM7UFIYkiZ/du93I7Y82IB2D15Nzsn7qxRnu72d3eea6JG/05HYWnE0irz9MW/U3VoAKGUh/l6GNd90/cBNZv7YdUqGDrUziIdFweTJsHll1dv39/Sf2PKiinMWDODnZn2Ap3cJJkHTn8AESEpOom0O49dZFM3pVa7XKpR6wrMLvV5JZAgIs2MMfvLJxaRMcAYgISEBFJTU31SyMpkZWXVi3LUF435fBgD06e3ZsqUZIqKiu8FsXy3fBttyGb+x7s4ocdGu3opsLZm+W9ctpGNqWX3zw7PZk/qHrvuUM3zdLt/DGX/DVcDe2uQaQX7/7jgR7IiXX8fv9a8rD8u+BFauD649l+1eNWxX7W1OKfu9nd3nmuipv9O2VQ9CaxP/p2qQQMIpTzNh30g8vflc2DeASRQaH5l82rtM2uW7e+QnQ19+sDHH9umS1VJ3ZrK//vu/7F4x+KSdSfGnMg1J1/D1d2urlXfC6VKiQQOl/pc/D4K+FMAYYyZDEwGSElJMQMGDPB2+aqUmppKfShHfdFYz4cx8MADdlAKh8PW9F59NSxbtpZlc9tz44/NOLIwmNf+CGbECCjsXUjuRbk1OkZw82CCE4KBY/sHRAYQlmxroZ2FTrJXV/1jtDR3+0uAENE5oiRN9vfZOAucFWXxJxXtH94hnAWLFzBgwAByp+VSmFlYo7KGdwjHEWzvscX7h54QSmCM/Vmb3yWf/Hvya5Snu/3dneeaqMm/0/Jly0npk1Jlnr74d6oODSCU8iRjfNqJOrBpIN0+6Ub2umyC44KrTP/887a/A9gmS5MnQ2hoxemLOzwDFBQVsHjHYqKCo7i629Vc3/N6Tks6TQMH5SlZQOnZRYrfH/FDWZSqtRdftEtQEMyYAcOHQ+62XCK+SeeO8S14aVEznn7aBhaJiXDWWYFEdqvd6HkAgVF/3t8R6KhTnhXtH96xbvPvuNs/tHUlN6FqcLd/cHwwwfFV3xMr4m5/d+e5Jqr8d8qgxvl769+pOjSAUMqTfNyB2hHkIO6SOLik8nTGwBNP2EXEBhL33ee+o3Shs5DZ62czcelE4sLjmPV/swAY2GYg0y6bxtBOQ4kMrv1FVKkKrAV6AB+4PvcA9rprvqRUffXrr/Doo/b9++8faxp6dM1ReBl2p+3iqS+aUVAAL7xgH+SsXFn9vmdK1RcaQCjlSX6YA6IqxsBDD9mgweGAKVPsTau8gzkH+fcv/+b1Za+z/fB2AJqENuFI3hGiQqJwiIOR3Uf6tvCqwRORQOy9JgAIEJFQoNAYU77Nwn+BKSIyHTsK03hgii/LqlRd5OXZWoWCAvjLX8r2KzNFtmZaHPapzVNPwfffw7JlcOed8O67/iixUrVXf37lKHU88GEAsf257ay9ci2ZyzMrTGMM3HOPDR4CA+0TsfLBQ/rRdB6Y+wCtX27NQ988xPbD22kf256Jgyay7Z5tRIVEefmbqOPceCAHO5rSta7340WktYhkiUhrAGPMHOA5YD6wHdgG/NU/RVaq5t58E9atg44d7TW3tOIAAtfgfEFBMH06hIXBtGmwZIlvy6pUXWkAoZQnFQcQPhjCdc87e0ifmU7hgYo7nz31lB2aNTgYPvrIzu9QXnZBNi8veZms/CzOa3MeX17zJevvWM+dfe8kOiT6zzsoVQPGmAnGGCm3TDDGbDfGRBpjtpdK+5IxJsEYE22MuVEnkVMNxaFD8Pe/2/cvvGADg9JKaiACjrUbbd/ePuAB2zetuPucUg2BBhBKeZIPh3DtMa8HbV9qS5Nz3TeenToVxo+3/RxmzIBLL7Xrf8/4nUe+eQTjulud2OREXhn0CstvWc686+YxuP1gHKKXBqWUqq5nnoEDB+Dss+Hii90kcD1bKh1AgG1e2qwZLFgAn3/u/XIq5Sn6K0EpT/JhE6bQE0I54d4TcAT++Vhz58Lo0fb9xIm2Le7GAxu5/pPr6fJGF5754Rk+Wf9JSfqxfcbSO7G318uslFLHm0OH4PXX7fvnnnM/OEVJE6Zyl+uYGPugB+DJJ7UWQjUcGkAo5Uk+aMJknKak9sCd1avtsIGFhbZa/NJrtzH609F0eq0T/135XxziYMwpY0hJrHq8aaWUUpWbPBmysmDgQDj1VPdp3DVhKjZmDMTG2n4Qixf/abNS9ZIGEEp5kg+aMB2cd5ClHZay61+7/rQtM9MGD1lZcM01ED7o73R4rQNv//o2ADf2vJG0O9KYdOkkTog5wWtlVEqpxiA/H155xb6///5KErpuDe4CiPBwGDvWvn/hBc+WTylv0QBCKU/yQROmvdP3krMxh4J9BWXWG2ObLW3YAN27w1tvQVhwCPlF+Vzd7WrW/WUd7wx9h5OanuS1simlVGPywQewezd06QKDBlWczjjdN2Eq9pe/2MEuZs+GjRs9X06lPE0DCKU8yctNmIqyi8j4OAOA+Gviy2x79VXDhx9CaEQ+M2faUUDu6nsXy29ZznvD36N9s/ZeKZNSSjVWkybZ13vucd/3oVhlTZgAWrSwtcbG2Ic/StV3GkAo5UlebsKU8WkGRVlFRPeLJrzdsanq3/tqE/fcZ4dzDRw2hrikgwCEBoZq52illPKC9eth0SKIiIARI6pIXEkTpmK33GJfp0yxk9EpVZ9pAKGUJ3m5CdO+6fsAiB9pax8O5R7iL588yMiRYIqCCD1jEi/ee7pO/qaUUl72zjv2dcQIiKriklvchKmyAOK006BTJ9i7F7780lOlVMo7NIBQypO8GEDkZ+RzYM4BCIC4K+OYumIqHV/ryBvPtIKDbYk7aSdbP/8/xvQeQ6Aj0OPHV0opZRUU2Ll2AG6+uer0TQY0gXHHHv64I3Isr7ff9kAhlfIiDSCU8iQv9oFI/yAdU2iIvSCWoOZBvLzkZfat7QhL7iYw0DDvoyQSYpp6/LhKKaXKmjMH9u2Dzp2hX7+q00d0joCLockZ7if+LDZqFAQGwhdf2PyVqq80gFDKk7zYB2LPtD0AJIxMIMARwEvn/Iv4b78A4JFHhJ49PX5IpZRSbsyYYV+vu67yztM1FR8PF15on0V9+KHn8lXK0zSAUMqTvNSEaW7qXLJ+zKIgpIBmQ5sB8Nm/+rJvRxQnn3xsJlOllFLedfSoHW4VqtF52uXIr0dgNhz+6XCVaa++2r4WBylK1UcaQCjlSR5uwpR+NJ3rPr6O//79vwCs7L7n1Q7dAAAgAElEQVSSnKAcli2Dl1+2h/nPf+z44Uoppbzvs88gO9s2XTqpmtPqHJx3EF6GjFkZVaYdOtQOw/3DD7B9ex0Lq5SXaAChlCd5qAmTMYZ3V75L59c7M23lNC5YfQEA1z12HRFBUdx1lx0v/L77oLeO0qqUUj5TXDNQXFNQHZE9I+FSiO4XXXXaSLj0Uvv+/fdrUUClfEADCKU8yQNNmAqKCrhkxiWM+mQU+3P2c03ANbROb01Q8yCaD2rOe+/BTz/ZiYcee8xD5VZKKVWlI0dsB2oRuPLK6u8Xe0Es3AfNhzevVvri4ET7Qaj6Ssd6VMqTPBBABAUEkRyTTNPQpvzzwn9ybadrOdj9IAUHCsjOc/DQQzbd009XPfa4Ukopz5kzB/LzoX9/aNnSe8e54ALbjGn5cti1C1q18t6xlKoNrYFQypNq2QfiYM5B1uxbU/L5ufOfY83YNVzf83oCQgOIGxpHyxtb8tRTsHs3nHqqHe5PKaWU7xR3nh46tGb75e7IhXWQtzuvWunDw20QAfDppzU7llK+oAGEUp5Uiz4Q8zbN4+Q3T2bIjCEcyTsCQERwBIlRiWXSbdoEL75o30+c6LXJrpVSSrlRUGDnZ4CaBxC7J+2GsbDn7T3V3qf4GMVBi1L1if4EUcqTatCEKbsgmzu+vIMLpl3AriO7SIhM4HBe2SH+0u5IY+2ItRxdd5RHHrFV56NGQd++3ii8UkqpiixcCIcOQadO0KFDDXd23RokoPqTRlxyib2VfPcdZGbW8HhKeZkGEEp5UjWbMC3ZuYRek3rx+rLXCXQE8uS5T7LwxoUkRScdyyrfyd5pe0n/Xzrrfxc+/BBCQuDJJ735BZRSSrlT2+ZLAKbI2Dc1+NXVvDmcfrqt+Zgzp+bHVMqbNIBQypOq0YTpxcUv0v+d/qTtT6Nr864sHb2UR898lEBH2TENHMEOUlak0OHfHXjinXAAxo6FpCR3uSqllPIWYzwTQNSkBqL0sbQZk6pvNIBQypOq0YQpuUkyTuPk/tPuZ/mY5fRq2avCtGHJYew4OZHPPrOd6h5+2NMFVkopVZVVq2DbNkhIqGUT0lo0YYJjAcSXX9qaCKXqCw0glPIkNwGE0zhZumtpyefhXYazduxaXrjgBUIDQ91nU+jEGPvEqniuh7vvhvh47xRbKaVUxYprAC69tHYDWNSmCRNA+/bQubPte7FgQc2Pq5S3aAChlCcVN2Fy9YHYfng75/33PPq/059f9vxSkqxz886VZvPH23+wtNNSUh/by7x5EB0N48Z5rdRKeY2IxIrIxyJyVES2icg1FaQLEZF/icheETkgIp+JiI5+r+qF4gBiyJDa7V/bJkygzZhU/eTRAKIGN4oJIlIgIlmlljaeLItSfuGqgTAOYeqKqZz85snM3zqfpqFNOZBzoNrZ7J2+l5y0HGb9z9507r8fYmO9UmKlvO11IB9IAEYCb4pIVzfp7gZOA7oDicBB4FVfFVKpiuzYAb/8YpuRnndeLTOpZRMmKBtAuCqmlfI7T9dAVPdGAfA/Y0xkqWWzh8uilO85naSHw/Aua7hh9g1k5mUytONQ1oxdw3ltqnfnyd2Wy+GFhyHYwTsb4oiNhXvu8XK5lfICEYkAhgOPGWOyjDGLgE+B69wkPwn42hiz1xiTC/wPqOj+oZTPFM/9UDw7dG3UtgkT2IlDW7SA7dth9eraHV8pTwusOkn1lLpRdDPGZAGLRKT4RqFdP1WjsOjQKoaPhX2R6UQFRzFx8ESu73E9ItV/6rR3xl4A1sc2I/uPQB640zZhUqoB6gAUGmPSSq1bCZztJu3bwCsikggcwj6E+qqijEVkDDAGICEhgdTUVE+VudaysrLqRTnqi+PlfLz3XlegOW3a/E5qavUngitjp33ZsHEDG1I31Hj3nj07MmdOS954YxMjRuyoXRnqmePl78NTGtr58FgAQc1uFACXisgBYA/wmjHmTXeJ9CZR/+n5OCZvYzq5gXD6/ijuvGgyLQ614Pvvv69ZJpPty3//SCAkpIhevX4iNbXhDr+hfx9lNbLzEQmUnwLrMBDlJu0GYAewCygCVgN3VJSxMWYyrv8tKSkpZsCAAR4obt2kpqZSH8pRXxwP56OwEFautO/vvLMjyckda5XP+v+u5w/+oGOXjrQc0LLG++/da+eC2LChLQMGtK1VGeqb4+Hvw5Ma2vnwZABRkxvFB9gL/16gLzBLRA4ZY2aUT6g3ifqvsZ+PX/f8So8WPXCIA7KyWHzTs3Q+9Uwcg0bUOK+sVVks37Kc3OBAluXHcvsYB0OH9vdCqX2nsf99lNfIzkcWUL7+LBo44ibt60AI0Aw4CjyIrYHQedeV3yxZYmeB7tABkpNrn09dmjCB7XvhcNjZsLOyIDKy9mVRyhM82Qei2jcKY8xvxpjdxpgiY8xi4BXgCg+WRSmvyyvM48F5D9J7cm9eXeLq6+l00jUdHI7KZ6KuyN5ptvnSvIJ4TICD++7zVGmV8os0IFBE2pda1wNY6yZtT2CKMeaAMSYP24H6VBGJ80E5lXJr7lz7euGFdcun6blN4VII7xheq/2bNbN9IQoKYP78upVFKU/wZABRkxtFeQao+dAESvnJyj9WkvLvFJ5f/DwiQmaeq/Kt3DCuNWGchn0z9gEw18QzYkTdnngp5W/GmKPAR8DfRCRCRPoDQ4F33SRfBowSkRgRCQLGAruNMRm+K7FSZRUHEBdcULd8WlzfAu6DmNNiap3HoEH2dc6cupVFKU/wWBMmY8xRESm+UYzGPk0aCpxePq2IDAUWYDvK9QHuAh71VFmU8pYiZxHPL36ex+c/ToGzgHax7Xj3snfpl9TPJqjGTNQVObTgEHk789grIaw1MUx/0IMFbwAyMzPZt28fBcf5dKsxMTGsW7fO68eJiIggKSkJR21mvfKsscA7wD5gP3C7MWatiJwJfGWMKW6MMQ6YiO0LEQysAS7zQ3mVAuDgQVi6FIKCoD60Ohw0CCZMgK++ssO51mBsDqU8zpN9IKD6N4oRrnQh2LEJnjXGTPVwWZTyqPSj6Qx9fyg/7vwRgNtTbuf5858nIjjiWKI6BBD7ptvah3kmgUGDhe7d61zkBiMzM5O9e/fSqlUrwsLCajRqVUNz5MgRoqLcdQ3zHKfTya5du8jIyCDez9OXG2MOAMPcrF+I7TtX/Hk/duQlpeqFb7+1l/Szzqp7n4PstGxYDwU9CghqGlSrPFJS7HxAW7bAxo12lmql/MWjAUQNbhRXe/K4SvlC07CmFJkiEqMSeXvI2wxqN+jPiYoDiBo2YXLmOdk3Mx2Ab0jgrUbW92Hfvn20atWK8PDatQ9WZTkcDhISEti2bZvfAwilGipPNV8C2PzQZvgEDsUfovnlzWuVR0CALcv779tmTBpAKH/ye922UvXZ9sPb2ZtlOzYHOgL54IoPWHP7GvfBAxzrA1HDGoj9X+6n6FAhG4gkvHMEAwfWpdQNT0FBAWG1naFJuRUUFERhYaG/i6FUg2QMfP21fe+JACKsXRh0gMCmdXtuq/0gVH2hAYRSbhhjeOfXd+j2RjdGfzYaY+wQfCc2OZGmYU0r3rGWTZhiB8fyXqeuTCGZO+5onG1bj+dmS/6g51Op2ktLszM/x8VBr151z6/t821hEjQ9p5L7RzUUBzPz50Nubt3LpVRtaQChVDl7juxhyPtDuPnTmzmSf4QgRxC5hdW8UtcygFi1LoB/r2/Omug4Ro2qYYGVUkp5VHHtw/nn16pLm9e0bAk9e0JOjp0TQil/qUf/LZTyL2MMM1bPoNub3fg87XOahDZh2mXTmPV/swgLqmbzmloM42qM4VXXNBI33qgTBCmllL95sv8DuCaSc3omL23GpOoDDSCUApzGyfAPhnPNR9dwIOcAg9oNYs3taxjZfWTNmoLUogbil8FrSJr6G83J5S9/qWHBld+lp6czduxYkpOTCQkJISEhgYEDBzJv3jwAkpOTeeGFF/xcSqVUdeXlHZuszVMBxMoLV8JAODDvQJ3zKp7UznWJUcovPD2Mq1INkkMctI9tT3RINC9e8CI397q5dm3IaxhA5Kfnkzl3P2cYYdV5HXRUjQZo+PDhZGdn8/bbb9OuXTv27dvH999/z/79+/1dNKVULSxeDNnZ0K0bJCZ6KFNX5bQE1L1v0mmnQUQErF4Ne/bYZk1K+ZrWQKhGK21/Ggu2LSj5PGHABH4b+xujTxld+w6oNRzGNSA2mAda9uUpOnPrfRrPNzSHDh1i4cKFPPPMMwwcOJATTzyRPn36MG7cOEaMGMGAAQPYtm0bDzzwACJS5u9q8eLFnH322YSHh9OqVStuv/12MjMzS7YPGDCA2267jbvvvpumTZvStGlTHnjgAZxOD7WDUEq55enmS+BqwoRnAoiQkGMT22kthPIXDSBUo1NQVMAzi56h+5vduXrW1RzOPQxAWFAYraJb1S3zGg7jOncu/Lw7jO0nxZdUS6uGIzIyksjISD799FNy3QyJ8tFHH5GUlMTjjz/Onj172LNnDwCrV6/mggsuYMiQIaxcuZKPPvqIFStWcNNNN5XZf/r06TidTn788UcmTZrE5MmTefnll33y3ZRqrIoDCE9ek43TBhCe+tVVHNwUl1UpX9NHnqpRWbxjMWO/GMvKvSsBOL/N+RiM5w5QgyZMznwn/54sgDB6dP0a6cPv/DUEqanZ30JgYCBTpkzhlltuYfLkyfTq1Yv+/ftz5ZVX0rdvX2JjYwkICCAqKooWLVoAdibq559/nquuuor777+/JK8333yTXr16sW/fvpLJ31q2bMnEiRMRETp16kRaWhovvfQS993XyGYaVMpH9u2DX36xT/nPPNODGXuwCRMcCyDmzbO3Hb1/KF/TPznVKKQfTeem2TfR/53+rNy7kuQmyXx97ddMGTaFJqFNPHegGgQQa8dtZegnyzjNsZ8bb/RcEZRvDR8+nN27d/PZZ58xePBgFi9eTL9+/Xjqqacq3Ofnn39m2rRpJTUYkZGR9O/fH4BNmzaVpOvXr1+ZZk+nnXYau3btKtPUSSnlOd98Y1/POgs8ObdlcQ2EpwKIjh3hhBNswLNqlUeyVKpGNIBQxz1jDBe/dzH/WfEfggOCGX/meNaOXcsFbT3YwLVYNYdxNU7D7nf3ciLZnHKaQzvBlWeMf5ZaCg0N5fzzz+fxxx9n8eLF3HzzzUyYMIH8/Hy36Z1OJ6NHj2bFihUly8qVK9mwYQM9e/asdTmUUnXjjeZLcKwPhKd+dYloMyblX9qESR23jDElHVcnDJjAK0te4dXBr9KhWQfvHbSaNRCHFh0m5FAe+wjhooc9WAOi6oUuXbpQWFhIbm4uwcHBFBUHli6nnHIKa9eupV27dpXms2TJkpK/Y4CffvqJxMREoqOjvVZ2pRorY7zTgRrweBMmsGV8+21b5gcf9Fi2SlWL1kCo484fWX9wy6e3cM+ce0rWXdT+IuaMnOPd4AGqHUD8+vxeAJZGxXPhYD+191d1tn//fs4991ymTZvGqlWr2LJlCx9++CHPPfccAwcOJDo6muTkZBYuXMiuXbvIyMgA4KGHHmLp0qXcdttt/Prrr2zcuJHPP/+cW2+9tUz+u3fv5p577uH3339n5syZPP/889x7773++KpKHffWrDk2LGq3bp7N29NNmAAGDrQ1EQsX2mFnlfIlrYFQx42cghz++dM/eXrR02TlZxESEMIjZz5Ci0jbebXWQ7PWRDWaMDnzneTPTScUSBiZUJNJq1U9ExkZSb9+/XjllVfYuHEjeXl5tGrVimuuuYbx48cD8Le//Y1bb72Vtm3bkpeXR2ZmJt27d2fBggWMHz+es88+m6KiItq0acNll11WJv+RI0dSVFRE3759ERFuvvlmDSCU8pLStQ+evl14ugkTQLNmkJICy5bBggXHZqhWyhc0gFANntM4eX/N+zz8zcPsyNwBwJCOQ3juvOdKggffFabqGoitHxwgNL+QzURw1SORPiqY8oaQkBCeeuqpSjtM9+vXj5UrV5Z8PnLkCAApKSnMmTOn0vwDAwN57bXXeO211zxTYKVUhb7+2r56vPkSeKUJE9iyLltmgx8NIJQvaQChGrTcwlzO+s9ZLNu9DICeLXry4gUvcu5J5/qnQNUIIFa9tJcmwI4O8bRu7ZtiKaWUqlhOjn2KD3D++Z7P3xtNmMAGEE8+qR2ple9pAKEatNDAUNo0bcPOzJ08ee6TjOoxigCHH9sEVRFAFGYWErFiPwBd7krwVamUUkpVYuFCyMuDU06B5s09n3/EyRHkSA6OMM92Pe3XDyIjYe1a2LULWtVxLlSlqks7UasGZcUfK7h0xqV8u/nbknUTB09k410bubHXjf4NHqDKPhCr38ggyDhZExDDJTeF+rBgqqFJTU3VpktK+YhXmy8B3WZ2g39BaGvPXveDg+Gcc+z7efM8mrVSldIAQjUIS3Yu4dIZl9JrUi8+T/ucJxc+WbItPiKe8KBwP5aulCpqIDZPtqMvZfVN8OgkRUoppWrPW/M/+ILOB6H8QZswqXrt+63f84+F/+CbzXZ60LDAMG7tfSuPnPmIn0tWgUoCiJxdeTTdcpAChH4PeaGOXCmlVI3t3m2HcI2IgNNO884xnIVOcFJmXhdPKQ4g5s2zt6AqRhFXyiM0gFD11tQVU7lh9g0ARAZHckefO7j3tHuJj4j3b8EqU0kTpiXPpOMAfg1rxr2XBPm2XEoppdwqfnI/YACEhHjnGD+d8BP8Afk78wlp5dmDtG8PJ54I27bBihW2H4dS3qZxqqo3juQdYdmuZSWfh3QcQlJ0EhPOnsC2e7bx9HlP1+/gASqtgZhxpCUT6ELRFSfoEyKllKonfNJ8SVyLF7rpiWgzJuV7+jNG+d2G/RsYN3ccSf9MYsj7Q8grzAOgaVhTtty9hb8O+CuxYbF+LmU1VRBA5OTA+x8H8D3xXPpIjB8KppT/iEisiHwsIkdFZJuIXFNJ2lNEZIGIZInIXhG525dlVY2L03ms87G3OlADnL77dPgOQlp4p4pDAwjla9qESflFflE+s9fPZtLPk/h2y7ERlXok9GDv0b20jrETJAQ6GtifaAUBxGefGjIzhd69oXNnP5RLKf96HcgHEoCewBcistIYs7Z0IhGJA+YA9wIzgWAgycdlVY3Ir79CRga0bg0dOvi7NLV37rn2trNoERw9avtzKOVNDezXmToebDu0jb5v9WXvUTsiUVhgGCO6jWBsn7GkJKb4uXR15KYPhDEGxiznESJJHN4O0P4PqnomTJjAzJkzWbNmjb+LUmsiEgEMB7oZY7KARSLyKXAd8HC55PcBXxtjprs+5wHrfFZY1eiUbr7k4b7NPhUbC336wJIl8P33cNFF/i6ROt5pAKG8Lv1oOgu2LWB4l+EAtI5pTZPQJsSFx3Fr71u5rsd1NAlt4udSeoibGoidqUeJzzzKKRRwxvX6X+54csMNN5CRkcHnn3/ulfzHjRvHnXfe6bPjeUkHoNAYk1Zq3UrgbDdp+wGrRWQx0A5YAvzFGLO9fEIRGQOMAUhISCA1NdXT5a6xrKyselGO+qIhnI8PPugBNKVVq7WkpqZ770B/gaLcIlJfTwUvTQHUoUMyS5Yk8847OwkP3+idg3hQQ/j78KWGdj7014zyisO5h/k87XP+t/Z/fLXxK4qcRWy9ZyutY1ojInx/w/fER8R7fDg7v3MTQMxeG8kznMqQlFyuSDzOvq/yqsjISCIjI/1djLqKBDLLrTsMRLlJmwScApwPrAaeA2YA/csnNMZMBiYDpKSkmAEDBniuxLWUmppKfShHfVHfz0dWlp3B2eGAu+7qStOm3jvWgi0LIAfOPOtMAsK9M+FpYCC8+y789lsSAwbU/5Z/9f3vw9ca2vnQTtTKY7ILspm6YiqXzriU+Bfiufbja/ks7TOMMQxuP5gjeUdK0iZEJhx/wQO4bcL07ruwi3DOvK+BdARXHrF9+3Yuu+wyoqKiiIqK4vLLL2fnzp1l0jz99NMkJCQQGRnJqFGjeOKJJ0hOTi7ZPmHCBLp161byfurUqXzxxReICCLSUJ5WZQHR5dZFA0fcpM0BPjbGLDPG5AJPAKeLiI48oDwuNRUKCuDUU/Fq8ABgiox948VfXX37QlQUrFsHO3Z47zhKgdZAqDowxpT0YwAoKCpgzOdjyC/KRxDOPvFshncezv91/T8SIhP8WFIfKlcDsX5lEUuXBhAVBUOH+rFcyqecTidDhw4lLCyM+fPnA3DHHXcwbNgwvvvuOwDef/99nnjiCV577TXOOussZs2axTPPPEPTCn7JjBs3jnXr1nHgwAHeffddAGJjG0RQmgYEikh7Y8wG17oewFo3aVcBptRn4yaNUh7x9df21ZujLxUrDiAkwHsPzoKCbGfq2bPtyFI33eS1QymlAYSqmcO5h1m4fSFzN83liw1fcCDnADNPnQlATGgMD/V/iFZRrRjWaVjjCRpKKxdArBj5O29zlPVndiA8XB+i1oQ8UfGNdtIlkxjTewwAk3+ezK2f31phWvPXY79Be0/uzS97fqkyXV19++23rFq1ik2bNpXUKLz33nu0a9eO1NRUhgwZwiuvvMINN9zA6NGjAXjkkUeYP38+aWlpbvOMjIwkLCyMkJAQWrRo4bGyepsx5qiIfAT8TURGY0dhGgqc7ib5f4BZIjIRG2A8Biwyxhz2WYFVozFnjn316vwPxVy3Bm8GEGCDodmzbedwDSCUN2kAoaq09dBW/rX8X3y35Tt+3vMzTuMs2dYsrBm7c3aXfP7bOX/zRxHrj+ImTA4HBZmFNP0tgxY4Sb4m2L/lUj61bt06EhMTyzRHatOmDYmJiaxfv54hQ4awfv16brnlljL79e3bt8IAooEbC7wD7AP2A7cbY9aKyJnAV8aYSABjzHci8ijwBRAOLAIqnDNCqdrauNEuTZvapj/eZIwpqUsTh/cDCLA1EEVFZVrTKuVRGkCoMnZl7mLJriUEOgIZ0nEIYGeIfvaHZwE7L0O/pH6cm3wuF7W/iFNbncrCBQv9WeT6pbgGIiCAxc9nEGKcpAVHM/rqMP+WqwGqbo3AmN5jSmojqvLzmJ/rUiSPOC77/lTBGHMAGOZm/UJsJ+vS694E3vRR0VQj9dVX9vWCC7z/I9sX/R+KtW0LJ50EW7bYOS5SGvjI6Kr+0gCiEcvMy2TlHytZsmsJP+38iSW7lrAz03by7JfUrySA6BrflcfOeozTTzidM1qfQWRwgx8VxntKNWHaNWUfiUDB2Qnl55VTx7nOnTuze/dutm7dWlILsXnzZnbv3k2nTp0A6NSpE8uWLeOmUu0Mli5dWmm+wcHBFBXXcimlaq04gBg82AcHK66098F9QMQGRZMm2WZMGkAob9EAohFwGiebD25m1d5V9EvqR2JUIgCPffcYE5dOLJM2OiSaU1udylmtzypZ5xCHNk2qLlcAkbk/mISdByhEOOvx5n4ulPKmzMxMVqxYUWZdu3bt6N69OyNHjuSVV14B4M477+SUU07h7LPt9Ad33303N954I3369OHMM8/k448/ZsmSJRV2ogZITk7mq6++4vfff6dZs2bExMQQFKQTEypVEzk54BrbwCf9H3xZAwFlA4hHH/XNMVXjowHEcabQWciHaz/k9/2/8/v+31mfsZ60/WlkF2QDMHXYVEb1GAVAn1Z9OKXlKfRJ7EPfVn3pl9SPjnEdcYg+Lq8119PhBZ+0JBIna6NjOe8M7f9wPFu4cCG9evUqs2748OHMnj2bu+66i3POOQeA8847j1dffbWkCdOIESPYvHkzDz/8MNnZ2Vx++eXcdtttzJ49u8Jj3XLLLaSmppKSkkJWVhbz589vUOOGK1UfLFgAubnQqxf4YjwCXwcQ555rx/H44Qc4csQO7aqUp2kA0YDkF+Wz/fB2th7aWmZxiIP/XvZfwNYW3PzpzeQU5pTZNzEqke4J3WkW1qxk3bXdr+Xa7tf69Dsc91w1EEeWBxFJHuFD4/1cIOVNU6ZMYcqUKRVu/+STT/607siRY9MfPProozxa6hHhZZddRrt27Uo+T5gwgQkTJpR8bt68OXPnzq1boZVq5HzafAl82oQJoEkTOO00G0B88w1cdplvjqsaFw0g6gGncZKRncHuI7vZc2SPfc3aw7BOw+gWbyeRenbRszzy7SMYN8OiRwVHYYxBRHCIg9tSbiMkIIROcZ3oGNeRjs060jTMy7PkKMvpZBddaXk0j2wCuHBCnL9LpOqp7Oxs3nzzTQYNGkRgYCCzZs1i9uzZzJo1y99FU+q45usAwtc1EAAXX2wDiC++0ABCeYcGEF6wP3s/f2T9weG8w+zP3s/+nP1kZGeQkZ1BbFgsD/Z/EIC8wjyS/pnEgZwDZYZGLdYysmVJABEXHoeI0Dq6NclNkv+0lPbShS95/TuqChQV8QO3Ew9sTozjojY6hp5yT0T46quveOqpp8jJyaF9+/ZMmzaNy/Rur5TXbN4MaWn2KX2/fj46qBMIcC0+cvHFtv/DF1/YinEdyEN5mkcDCBGJBd4GLgAygEeMMe+5SSfAM8Bo16q3gIeNMT6fdXRv1l4O5x3maP5RsguyyyxJ0UmceeKZJemeX/w8mXmZbNyxked2PcfhvMNk5mVyOPcwH1/1Mb0TewPw+PzHeWP5G26P16V5l5IAIiQwhKP5R3EaJ7FhsSRGJdIysmXJ68kJJ5fsd233axnVYxRBAdphsj4ryA0ihK4AtLi2EU6kp6otLCyMb775xt/FUKpRKa59OP98CPTRI9SgZkEMKBxAamqqbw4InHwyJCXBzp12ONfevX12aNVIePq/z+tAPpCAnW30CxFZaYxZWy7dGOyY4D2w06vMA7YA/6os811HdnHHl3eQX5RPXlEe+UX59n1hHtf3uJ4ru14JwNcbv2bcvHEl28qnT38gneiQaACumnkV32/73u3xRtYKDtQAACAASURBVHQbURJAHMk/wos/vnhsY3rZtAdyDpS8PyHmBDrFdSI6JJq48DiahTUjLjyOuPA4Wse0LrPftnu20SS0SZWBQUhgSKXblf8ZY/jl54uJAdZLFNf+P202ppRS9YnP+z/4iYithZg0ydZCaAChPM1jAYSIRADDgW7GmCxgkYh8ClwHPFwu+fXAi8aYna59XwRuoYoAInRzKAOuGuB2W3RINIuDFwMQVBjEX3P+Snp0OmPHjC1J85/X/0NkTiTZN2QTnWQDiFFvjeKe1fcgIghS5jU4IJjFd9o8DYav87/GIQ5+ePYH+pzZh+iQaBx3OShaUkTHMztCW3ucqxdezVmvnkVFFrP4T+u6zOhCk7ObALDt6W3senUXrR9uTdJdSQDs/2I/v9/ye2Wn50/c7R97USyd3rLj0OfuyOWXvr/UKE+3+8cA646lWdp1KYUHC6udZ0hSCL2XHru6Fe/fZ00fgmJtYLV2xFoOLzhco7K629/dea6Jyv6dCg8Xkn44EQcBrOkbS2R045swTCml6qvc3GPDtw4a5N+y+ELpAOLxx/1dGnW88WQNRAeg0BiTVmrdSuBsN2m7uraVTtfVXaYiMgZbY0E7aUdcVgWdUrMgn3wAHDiII44m4U2Y0XcGgRJIkCOI6BejkSxh3fJ1rN+4HoA2eW0gs+IvVZwnQDB2OM7eob0J3xVOIf+/vfsOj6pKHzj+Pemd0BJC7ygiTRSpBsSOgooNRBEEhcUuyhZXfurqqqsusFgQBAEFWQUEBYVVooAghCqgodeQ0EJCSM+c3x9nMsmESTIJk8xM8n6e5z4zc+ece0/u3NyZ956WB8eBJNi9eTe7fXabhDuBEyVv05FtG7fZprovyL9vxz72xe0z6+LLv01H+ZP2JJEUl2TWJZV/m47yW/It9lWzxyj1mBaXY8lxmH/dmnUmOAHYX/6yOsrv6DiXR2mfk9bwDO3wJ5KnO6wgLu5w+TZeTaWnp5dZdV+rVi270Ymqs/z8/Cr7W7Oysqq02YQQnuzHHyEjwwzfGhNTdfvNTsxmxy07IBjYUHX77d8fAgNh40ZIToZoaVUrXMiVAUQYF/9sTAUcjUAcZn2vaLowpZQq3g9Caz0dmA5wVaerdI8VPZwukPJVBEQXjsGfnZANGgIaBKB8zN3h3O9ysWRf3IG5NOv/WG8be70gv18dP3yDTA+pvK555L9dvtliHeX3DffFL9x8RPnd88kb7fxdfcBhfp8gH9tdeUuehdy+ueXapqP86zestxuLPvuPbBwMFlWiiz6nP1zzOTnKX5mf07p1sC8HGnKcUXfVxlfG5wcgLi6uzLkKfv/9d8JryGDl58+fr7K/NSgo6KI5KoSoqZYuNY+DBlXtfi3ZFi7suABVMOdEUaGh0K8ffPedabo1YkTV7l9Ub64MINKBiGLrIgBHt9qKp40A0svqRK38FYENK94XIDDm4rz+dSvQKblIHYuj/H4RfvhFVPzQOsrvG+yLb3DFh3BwlN/Hz+eSjqctfx379Y6Oc3m47HMqI7+rP6e5c83jMD7D179rhbcrhBDCtSwWWLbMPL/jjqrdd2DDQLpt60b8lviq3TEwcKAJIL79VgII4VquHNhrD+CnlGpTZF0noHgHaqzrOjmRTgivkJ0NCxea58OZK2PmCSGEB9myBRITzchEnTtX7b59An0I6xQGLap2v2D6QQCsXAk5OaWnFaI8XPYrR2t9AVgEvKKUClVK9QIGAXMdJJ8DPKuUaqSUagg8B8x2VVmEqGrffgspKdApdC9XshN8Zf4H4Tr3338/Q4YMcXcxhPBaBc2X7rjDjFBUUzRvDu3bQ1oarF3r7tKI6sTVt0nHYboJnQTmA2O11ruUUn2UUulF0n0ELAN+w3RF/da6TgivVNB8aXj9780TqYGo9pRSpS4jpL2AEB6jaABR1bKOZfHHyD9gVtXvGwprIQqacAnhCi6dB0JrfRYzv0Px9WswHacLXmvgBesihFc7c8bUQPj4wNB6K+EQEkDUACdOFA7h9c033zB69Gi7dcHBwe4olhCimMOHYft2CAsDd4xtkXcmj6RZSdCy6vcNptP422/D4sXw7rs1qwZGVB75lSPEJVq4EHJzYcAAiPE9aVZKE6Zqr0GDBrYlMjLyonW1apkxhJ999lnatGlDcHAwLVq04JVXXiGnSGPkiRMn0q1bN+bMmUOLFi2IiIhgyJAhpKSkXLTPt99+m5iYGOrUqcPo0aPJzs6umj9WCC9WcOf95pvNsKZVTVus48O46RdXjx7QoIEJpLZtc08ZRPUjAYQQl8jWfGk4ZqgPkBoIYVOrVi3mzJnD77//zpQpU5g3bx5vv/22XZqEhASWLVvGsmXLWL58OevXr2fSpEl2aVatWsWhQ4dYvXo18+bNY8GCBbz//vtV+JcI4Z3cNXxrAZ1vDSDcdOffxwcGW9uGLFrknjKI6kd+5QhxCfbtg/XrISTEeoGWAMIllHLPUhlefvllevToQfPmzbn99tt56qmnmD9//kXpZs+eTYcOHejduzcjR47khx9+sHu/Xr16TJ06lcsuu4xbb72VwYMHX5RGCGEvNRXi4kyl8K23uqkQBdMNubFi+s47zePixe4rg6heXNoHQoiaZt4883jXXaZ9LfnWbwoJIITV/PnzmTp1KgcOHCA9PZ28vDwCAgLs0rRs2ZLQ0FDb64YNG3Ly5Em7NB06dMCnyHnVsGFDEhISKrfwQni5774zTUyvuw7q1Ck7fWVwdxMmMH0/IiNh1y5ISIB27dxXFlE9yK8cISpI68IAYvhw68qCGgjpA3FJtHbP4mpxcXEMHz6cO+64g2+++YatW7fy4osv2vWBAPD3t5/oUCmFxWIpdxohhL2vvjKP7mq+BO5vwgQQEGAmlQOphRCuIQGEEBX0yy+wfz/ExMD111tXShMmUcS6deto1aqVraN0mzZtOHLkiLuLJUSNcOECfPONee7WaVQ8oAkTmJpykABCuIb8yhGigj791DwOH16kwkGaMIki2rZty8GDB1m4cCH79+9nypQpfP311+4uVpVTStVRSi1WSl1QSh1WSg0tI32AUup3pdSxqiqjqH6+/RYyM80oRE2auK8cntCECeCmmyA4GDZuhKNH3VsW4f3kV44QFZCZCV98YZ4//HCRN6QJkyhiyJAhPPHEE4wbN47OnTuzdu1aJk6c6O5iucM0IAeIBoYBHyilrigl/QTgVFUUTFRfCxeax3vvdW85PKEJE5jBPm6+2TxfssS9ZRHeTwIIISrg668hLQ26dYP27Yu8IU2YaqQhQ4agHXSiUErxzjvvcPr0ac6fP8/ChQsZO3YsWVlZtjT//Oc/iY+Pt8v3+OOPc/r0advrBQsW8OWXX9qlcZTPEymlQoG7gZe01ula67XAUmB4CelbAA8Cb1RdKUV1k54Oy5eb525tvgQe04QJpBmTcB0ZhUmICihovmRX+wASQAhxsbZAntZ6T5F124HrSkg/FfgLkFnaRpVSY4AxANHR0cTFxV16SS9Renq6R5TDU7jzePz4Y30yM6/giitS2bdvK/v2uaUYhnXytjxLntvPj8hIP/z8evLTT4pFi9ZTp05O2Zkqify/2PO24yEBhBDllJgIK1eCvz888ECxNwv6QEgTJiEKhAFpxdalAuHFEyql7gR8tdaLlVKxpW1Uaz0dmA7QrVs3HRtbavIqERcXhyeUw1O483hMnWoeR4+u5fbP5HT6aXayEz9/P7eXBcx8GEuXwtGjPW01Eu4g/y/2vO14yG1SIcrps89MRcPAgVC3brE3pQZCiOLSgYhi6yKA80VXWJs6vQU8WUXlEtXU+fMe1HwJiOwbyVVbr4In3F0SY6h1CIPPP3dvOYR3k185QpSD1qU0XwIJIIS42B7ATynVpsi6TsCuYunaAM2BNUqpJGAREKOUSlJKNa+Ccopq4ptvICsLevWCRo3cXRrwi/AjvHM4eEBZAG6/3Ux8unEj7m3aJbya/MoRohy2bDEzedarB7fc4iCBDOMqhB2t9QVMMPCKUipUKdULGATMLZZ0J9AE6GxdHgWSrc9l0EnhtP/+1zy6e/QlTxUSAnfeaZ5LLYSoKPmVI0Q5FNQ+DB1qZva8iAzjKoQj44Bg4CQwHxirtd6llOqjlEoH0Frnaa2TChbgLGCxvs4vedNCFEpJMfM/KAV33+3u0hhpv6bxx8g/zNhjHmLYMPP4+eemZl2I8pJO1EI4KSen8G6Nw+ZLIE2YhHBAa30WGOxg/RpMJ2tHeeKAxpVbMlHdLFhgrtU33OAZzZcAMvdnkjQrCfq7uySFrr8eoqIgIQG2boWuXd1dIuFt5FeOEE5avhzOnIEOHaBLlxISSQAhhBBuM3u2eRwxwp2lsBfRPYJ2M9rBbe4uSSE/P7jvPvP8s8/cWxbhneRXjhBOKtp5WpU0o6gM4yqEEG6xe7fpGBwRAYMvqu9yn+BWwcSMigEPu8tfMBrTggWFX11COEsCCCGccOqUaVfr41PYdtQhqYEQQgi3KLjJc999pqOwKF337tCypZnbyIvmLxMeQn7lCOGEOXMgN9eMvBQTU0pCCSBqlBEjRqCUQimFn58fTZs2ZezYsaSkpDi9jbi4OJRSnD59usR9DBw4sNz5hKhJ8vJgrnVcrxL7qLnJhV0XOD7tuG1Gak+hFDz4oHk+c6Z7yyK8j/zKEaIMWsPHH5vnY8aUkViGca1xBgwYwIkTJzh06BAzZsxg2bJljBs3zt3FEqJGWbUKTpyA1q2hZ093l8Ze6tpU9o7fCz+4uyQXGzXKBBJffQVyL0KUh/zKEaIMa9aYkSoaNoRbby0jsQzjWuMEBgbSoEEDGjduzI033sh9993HypUrbe+npqYyZswYoqKiCA8P55ZbbiE+Pt6NJRai+ilovjRiRCl91NxE51vHSfXAX1xNm5qa9ZwcU9MuhLM88HQWwrNMn24eH3nEjFxRKmnCVKMdOHCA7777Dn9/fwC01tx2220cP36cb775hq1bt9KzZ0/69+/PiRMn3FxaIaqHlBRYssQEDsOHu7s0F/PkAAIKa9anT5c5IYTzZB4IIUpx9ix8+aV5PmqUExkkgHCZOBVXrvRhXcPotrnbRfljdaxtXfxV8aRvSXeYv2i68vjuu+8ICwsjPz+frKwsAN59910AVq9ezbZt2zh16hTBwcEAvPTSS6xcuZK5c+fywgsvVGifQohCc+dCdraZ26BpU3eXxgHr14KnBhC33Wb69iUkmBr3vn3dXSLhDTz0dBbCM8ybZ76YbrwRWrRwIoMM41rj9O3bl23btrFx40aeeOIJbr31Vp588kkANm/eTEZGBvXr1ycsLIywsDBiYmLYuXMn+/fvd3PJhfB+Fgv85z/m+dix7i1LSTy9BsLPr/AGWUGNuxBlkRoIIUqgdeHFdPRoJzMU8LRGuF6oojUCpeUvWkPhKiEhIbRu3RqAKVOm0K9fP1599VUmTZqExWIhOjqaNWvW2NKnp6cTFhZGRESEU9uPiIhwGGycO3cOHx8fwsPDXfOHCOGFVq6EvXuhSRMYNMjdpXHM0wMIMAHEP/5hatwnT4a6dd1dIuHpPPh0FsK9NmyAXbugfn244w4nMlibL2lpvlSjvfzyy7z55pskJibStWtXkpOT8fHxoXXr1rRu3ZpWrVrRunVroqKinNpeu3bt2L17N5mZmXbrt2zZQrNmzQgMDKyMP0MIrzB1qnkcN86JPmru4uFNmACaN4ebbjI17gXD4QpRGg8+nYVwrw8/NI8jRkBAgBMZrM2XtNQ+1GixsbG0b9+e1157jQEDBtCrVy8GDRrEihUrOHjwIL/++isvv/yyXa0EwM6dO9m2bZvdYrFYGDZsGH5+fjz00ENs3ryZffv2MWvWLP79738zYcIEN/2VQrjfvn2wYgUEBsKjj7q7NCXzhhoIKOxM/f77hd35hCiJh5/OQrjHyZOwYIFpifT4405mkg7Uwuq5555j5syZHDlyhOXLl9O/f39Gjx5Nu3btGDFiBAkJCTRs2NAuT79+/ejSpYvdkpGRQWRkJGvWrCE/P5877riDzp07M3nyZN59910ed/rkFKL6mTbNtBwdOhTq1XN3aUrmLQHE7bdDs2amSdjSpe4ujfB0nlrhJ4RbffyxGRd74EBo2dLJTNKEqcaZPXu2w/VDhw5l6NChtteTJ09m8uTJAJw/f96u30JsbCy6jLET27Zty6JFiy69wEJUE+np8Mkn5vkTT7i3LGUquJvv4ZXTfn7wzDPw9NPwr3/B4MHuLpHwZPJLR4hicnPhgw/M83J9MRWMwCRNmIQQolLNnQtpadCrF3Tp4u7SlM5WA+EFg/ONGgW1a8O6dbB+vbtLIzyZBBBCFLNkCRw/Du3awYAB5chYUAMhQ7gKIUSlyc+Hf//bPPf42gdAW7yjCRNAWFjhcLhvv+3esgjP5gWnsxBVq2BUj/Hjy9mdoSCAkBoIIYSoNIsXw549ZuSgu+92d2nKVv/O+rSb0Q6udXdJnDN+vBk4ZMkSc5yFcET6QAhRxPbtZibO8HB4+OFyZpZO1EIIUam0hjfeMM8nTHD/0K1aa5T1ptHZzLP8ffXfSUpP4nTGac5kniE1K5U8Sx55ljyebPEkscQCsGDnAj7a/BENwxvSKLwRjSMa06ZOG9rXb0+TWk3wUe77HomJgeHDYeZMeO+9wia9QhQlAYQQRRTMaDpihAkiykWGcb0kRb+IxaUrq2O2EN5o1SrYsgWiouCRR6p23xZtYfep3aw5vIZNiZvYmrSV5pHNWXzfYgACfQOZtmlaifmz8rNszw+mHCTuUJzDdHWC63BqwilbEHHywknqh9Sv0uvjc8+ZAGL2bJg0CaKjq2zXwktIACGE1cmTMG+eef6nP1VgA1IDUWH+/v5kZmYSEhLi7qJUG7m5ufi5+/asEC6ktfkxC/DssxAcXDX7/XbPt8zYOoOfD//M2cyzdu+dvHDS9jw0IJT3b32fOsF1qBdSj7ohdYkMiiRvQx65v+dyLO2YLe1DnR7i6kZXczztOInnEzmSeoSEMwnsPrWbqNAoW/CgtebyaZcTHhDOgJYDGNByADe0vIG6IZU7VfTll5sJVJcuNX0h/vWvSt2d8ELy7SKE1dSpkJVlhm5t164CG5BhXCssKiqK48eP06hRI4KDg6Um4hJZLBaSk5OpVauWu4sihMt8/70ZGahevQre5HHSzpM7qR1Um0YRjQBIOJPAkj+WANAovBF9m/WlR+MedI3pSsfojnZ5x1499qLt7V2+l+Spyfj/yd+2rlFEI9v2i8vIzbA9T0pPAuBw6mFmbp3JzK0z8VW+xDaP5a7L7+LeK+6lXkjlTIIxaZIJIKZNMzUSMTGVshvhpSSAEAIzpvg0a83ziy9WcCMyjGuFRUREAJCYmEhubq6bS1O5srKyCAoKqvT9hIaGUs+TZ9cSohy0hr//3Tx/8UUzWpArJZxO4ItdX/DFri/YfWo3f+vzN17t/yoAd11+l60GoHlk83Lf4KjVuxY6T5PYKtGp9CH+hTWxMeExnJpwim1J2/jfgf+xcv9Kfjr8Ez8c/IEfDv5A5wadbQGERVtc2neiSxe4807Taf2f/wTrVDZCAC4MIJRSdYCZwI3AaeDPWuvPS0g7CfgrkF1kdUet9QFXlUeI8vj4Y0hJgZ49oXfvCm5EhnG9JBEREbZAojqLi4uji6cPXC+Eh1m8GDZtMm3xx41zzTbTstOY/9t8ZmydQXxivG19neA6+PkU/jxqHtmc0VeNrvB+ou6NIureKBLjnAsgivNRPnSN6UrXmK680OsFUjJTWLZnGT8c/IFrGxcO7TR4wWB8fXx5pPMj3NL6Fvx9/UvZqnMmTTLH/sMPzSRzzZtf8iZFNeHKGohpQA4QDXQGvlVKbdda7yoh/Rda6wdduH8hKiQnB9591zyvcO0DyDCuQpTA2RtMSqkJwMNAM2u697XWMhp9DZebCxMnmucvvwyu6io1fvl45u6YC0BEYAR3XnYn911xHwNaDnDJj+/KUju4Ng91eoiHOj1kW5eSmcL3+78nJz+HJX8soX5IfYZ3HM7Yq8fSuk7rCu+rY0cYNgw++wz++lfzKAS4aB4IpVQocDfwktY6XWu9FlgKDHfF9oWoTPPnw7Fj0L696f9QYdKJWoiSFL3BNAz4QCl1hYN0CngIqA3cDIxXSt1fZaUUHmn6dNi7F9q2hUcfrdg2svOy+WTrJ/xy9Bfbukc6P0Js81jm3TmPpOeSmD14Nre0cc2d+6Iy92eS9msanHPpZu3UDq7NoacO8daAt7i83uWcyjjFuxvepe3Uttz2+W3sOVPxCR1ee83MC/H557B5swsLLbyaq2og2gJ5WuuiZ+h24LpS8tyulDoLnAD+o7V2ONKwUmoMMAYgOjqauLg415T4EqSnp3tEOTyFNx8PiwVefvlqIJTbb/+dn39OrvC2go8epTtg0dprj0dl8ObzozLUtONR5AZTB611OrBWKVVwg2li0bRa67eKvExQSn0N9AIWVFV5hWc5e7Zw5KU33wT/cv62P5Nxhg/jP2TqxqkkX0jmplY38d2D3wHQr0U/+rXo59oCO3D4tcMkzU6CCcDgyttPTHgME3pN4Pmez7MpcRMfxH/A/N/ms2r/KiICC5uH5lvy8fVxvqlt8+bw5JNmJKZnnoGffpKufgKUK8YKV0r1Af6rtW5QZN1oYJjWOtZB+vaYWDwZ6A58BTyrtZ5f2n66deum4+PjS0tSJeLi4oiNjXV3MTyGNx+Pr7+GwYOhcWPYv9/cZamwP/6Ayy8no0kTQo4ccVkZvZ03nx+VwVOOh1Jqs9a6WxXspwuwTmsdUmTd88B1WuvbS8mngC3AR1rrDx28X/Tm0lULFrg/xkhPTyfM1b17vZgrjsfkyW1YsqQRnTun8O67253+4Xo88zhfHvuS75K+I8ti5l9oFdqK+5rcx4CoAVU70tsbwErIejqLoEGVP4BCUam5qexM3Umver0AyNf5jIofRafITtzT6B4ahzR2ajvp6X4MH34N584F8Ne/7mbAgJNlZypzm/L/UpSnHI9+/fo59d3gVA2EUiqOkmsT1gFPAMV7P0YA5x1l0FrvLvLyF6XUZGAIUGoAIYQrmdoH8/z55y8xeCjYIDKMqxDFhAFpxdalAmVN1TgJ08x2lqM3tdbTgelgbi55QlDmKcGhp7jU47FtmxlG1NcX5s6tTYcOzm1rwc4FDP9qOBpzg/Tm1jfzXI/nuL7F9W4ZInr3jN2c5CRBwUFuOT8GMcj2fO2RtRz++TCHMw6zLHEZd15+JxN6TrDrjF2Sd96BUaNg1qz2vPhi+/JPtlqM/L/Y87bj4dQvHa11rNZalbD0BvYAfkqpNkWydQJK6kB90S4wbV+FqDJffQXbt0OjRvDYYy7YoAzjKoQj6ZTjBhOAUmo8pi/EbVrr7JLSieorP99cly0WGD8eOnQoJa0ln/1n99te92/Rn4jACB7p/Ai/jf2NFcNWMKBlFdc62BXQ+ugB95Z6N+3NrnG7GNVlFP6+/iz6fRE9Zvagz6w+LE1YikVbSsw7YgRccw0kJsLf/lZ1ZRaeySWns9b6ArAIeEUpFaqU6gUMAuY6Sq+UGqSUqq2Ma4Anga9dURYhnJGfXzim+EsvgUuG5ZcaCCEcKdcNJqXUSEzfiOu11sccpRHV39SpsHGjucHzyiuO01zIucB/Nv6Htv9pS/85/cmz5AEQFRrF8WeP88mgT+gQVUrkUUW0xdpU3EO+GtrXb8+MO2Zw6KlD/Ln3n4kMimTtkbU8ueJJ8i35Jebz8THDufr6ms9nw4YqLLTwOK48nccBwcBJTFOksQVDuCql+iil0oukvR/Yh7kDNQd4U2v9qQvLIkSpPvvMdFlo0QIeecRFG5UAQoiLlOcGk1JqGPA6cIPMC1RzHThghgwF+OADKD49zInzJ/jrD3+lyXtNeGLFExxIOYCv8uXQuUO2NKEBoVVX4DLofM8KIArEhMfw+vWvc+TpI7x303u82u9V2whUJy+c5PU1r3M286xdni5dTJNfrU1zpqwsd5RceAKXzQOhtT5LCeMLaK3XYNrBFrx+wFX7FaK8cnMLR/WYNMkFfR8KSBMmIUoyDvgEc4PpDNYbTNYBOFZorQu+H14D6gKbijQ3mae1fryqCyzcIz8fHnoIMjLgvvvg9iLd7M9lnePZ75/ls98+Iyc/B4BrG1/Lcz2e487L7izXyEJVyoOaMDkSHhjO09c+bbdu2sZpvPLzK/xjzT8Y1WUUz1z7DC1qtwBM38FFi2D3blOD/7bM1FIjeejpLETlmTULDh6Eyy4zE+S4jNRACOGQ1vqs1nqw1jpUa920YBI5rfWaIsEDWusWWmt/rXVYkUWChxrkrbdg3TqIiYFp0+zfCw8IJ+5QHLn5udx52Z2sG7mO9aPWM6T9EM8NHvC8JkzO6N+iPze2upGM3AymbpxK66mtue/L+9h0fBPBwTB3rmnK9M47UINGpRZFeNHpLMSlO3++sPbh//7PXABdRiaSE0KIClu/vrBv2scz8lhxfB7dZ3TnxPkTAPj6+PLJoE/Y88QeFt23iJ5NerqxtM7z1CZMpbmu+XV8/+D3bH98Ow91eggf5cPCXQu5ZsY1vLjqRbp3N83MtIYHH4RTp9xdYlHVvOh0FuLSvfEGnDhhRpIYMsTFG5caCCGEqJDTp+HeeyEvD3rfu4nHE1owfPFwNh7fyMdbPrali20eS+s6rd1Y0grw8CZMpekY3ZFPB3/KwacO8kLPF4gIjODGVjcCZiSmq6/N4fhxE0Tkl9z/WlRDXng6C1Ex+/eb6laAKVMqoaJA+kAIIUS55eXB4HvSOXYMfJpsYG27nhxLO0b7+u2ZecdMXuj1gruLeEm8sQlTcY0jGvPmDW9y7Jlj9G/RHzCzgtd96E+o0NOsXAnPTcxwcylFVfLi01mI8nnuvAVGiwAAH4JJREFUOcjJMR30unevhB1IDYQQQpTb88/DurgwCDmJZcg93NT2er4b9h2/jf2NkV1GEuRXtbM3u5o3NmEqSXhguG0+jey8bE77bUPf9QCoPCb/K4R+T83l91O/u7mUoiq4bBQmITzZqlXw9dcQFgb//Gcl7UQCCCGEKFNmbiaf/fYZV0ZdyZZl3Zk8Gfz8Ldz699m8MfJ72tdv7+4iulS9O+oR0iaExKhEdxfFpQL9Atn46EZWD1jNk34fsGvWE8T9537ap9zC9QNgyi1Tqt1nKQpJACGqvdxceNo6Qt3f/mZG96gU0oRJCCFK9MfpP/go/iM+3f4pKVkpdEt9nc3/NtXB0z/y4ZFHvLupUkkaP9kYgMS46hVAACil6N+iPzs/6c/IkNPMmlYPFn5FXK2+RN4ZaUuXb8n36JGyRPlJACGqvbfeMuNVt2pVGEhUCqmBEEIIOzn5OSz+fTEfbv6QuENxtvVtU55g+wcvoDW8+qoLJ/QUbjNjSj3OJ8OXX9Yi8r8buPBkMISDRVvo9GEnusZ05dGuj9KnaR+U3GjzehJAiGpt504zXCvARx9BYGAl7kyGcRVCCDvvrn+XP//wZwBC/UMZeuVQOl14nucfaUtuDowfXzjrdHWV/ls6lgwLZLq7JJXLxwfmzIHkZFizJpi+feGHH+BCrc3sPrWbXad2MXfHXNrWbcvIziNpk9PG3UUWl0B+6YhqKy/P3NXKzYXHHoPrr6/kHUoNhBCiBkvJTOHD+A+Zs32Obd3wjsPpGtOV9299n8TnErk+YzrPPtyWrCxzXZ48ufq3+kwYlcCWa7fAQXeXpPIFB8OKFdCvHyQlQWwsBJ65mn1P7uMvvf9Cw/CG7Dmzh4k/TOSe9fcwaMEgTl2QSSS8kfzSEdXWO+9AfDw0aWKaMVU66QMhhKhhcvNzWZawjHv+ew8N3mnA2G/H8urPr6K1GXmoUUQjNo/ZzNirxzJjWgT3329Gw3viCXj//ZpRYRt6RSjhV4dDiLtLUjVCQ+Gbb+DGG80Ec/36wfGdLfnH9f/g8NOHWfbAMgZfNhilFFtObKFOcB1b3t2ndtvOHeHZpAmTqJZ27y6c0XTGDIiIqIKdSg2EEKKG2HVyF9M2TePL3V9yKsPcQVYobmh5Aw91egiLttjS5ufDhAnw3nvm9VtvmaFba8q9lstmXQZAXFycewtShUJCzMiH99xjgonrr4epU+Gxx/wY2HYgA9sOZNHKRURdFmXrXH3ywkk6ftCRxhGNub/D/TzQ4QE6RneU/hIeSgIIUe1kZsKwYeYu16hR5i5IlZAAQghRTeXm55KSlUJUaBQAR1KP8EH8BwBcXu9yHu70MMM6DqNxRGO7fImJZpbi1avNxGOzZ8PQoVVdeuEOQUGweDG88IIJHh9/HLZtM83WAgKgTkAdejftbUu/7+w+GoQ14HDqYd5c9yZvrnuTy+tdzj3t72HQZYPo0qCLBBMeRH7piGrnqafMRap168KZp6uENGESQlQjadlpLPp9ESO/HkmDdxrw+DeP2967vuX1/L3v39kyZgu7xu3ixd4vXhQ8bNxYh86dTfAQHQ0rV9bM4MGSazGTydXAljl+fvDuuyZwDAyEDz+E3r3hdwdzzfVs0pMjzxzhpxE/8fhVj1M3uC6/n/6dV35+hWs+voaUrBRbWmnm5H5SAyGqlTlz4OOPzYXqv/+FWrWqcOdSAyGE8HIHUg6w6PdFLN+7nDVH1pBnybO9dyT1CBZtwUf5EOAbwP/1+z+H20hLM3PuTJ3aEYABA2DePBNE1ETxHePJ+CMDZru7JO7z8MNw+eUwZAhs2gRdusDIkY3p0wd8i0wP4aN86NusL32b9WXKLVP44eAPLPljCanZqba+EhZt4Yr3r6Bzg87c3Opmbmh1Aw3DG7rpL6u5JIAQ1cbOnaaKFGDaNOjcuYoLIMO4CiG8THJ6MhpNg7AGACxNWMqEVRMA82Oud9Pe3NL6FgZfNrjMWYW1Njdunn4aTpwAHx/Nq68qJk6s2ZdFnW+9W17DK6evuQZ++w2eeQZmzYIPPmjN9u0wZQpcddXF6f19/bm59c3c3Ppmu/Xbk7bzx+k/+OP0HyzYuQCA9vXbc0PLG7ih5Q30a9GPEP8a0mPdjSSAENVCSoq5s5GZae50jBzphkIU1EBIEyYhhIdKTk/mp8M/EXcojrhDcfx++ncm9prIGwPeAGBg24FsTdrKra1v5cZWN1I7uLZT242Ph7/8BVatMq+vvRZGjdrMo492q6w/xWvYAgiZiJlateCTT+Cuu2DEiGx++SWQbt1g+HD4xz/MqIll6RLThX1P7GPZnmWsOrCKnw79xO5Tu9l9ajeTf51M/Oh4rmpoIpK9Z/YSFRpFraCqbI5QM0gAIbxeVhYMGgQJCXDllWZoQLf8hi/oA1GTb7UJITzSy6tf5otdX5BwJsFufYh/CJl5hTOcta7Tmk8Hf+r0drdtg5dfhqVLzevateGf/4RHH4Wff053Sdm9XsGAVHJvyWbgQJg9exM//dSbKVNg7lxTe/X446aGomnT0vO3qtOKp699mqevfZqc/Bw2HNvAqv2r+PX4r3RuUNj8YOTSkaw7so4ro6+kd5Pe9Grai55NetKsVjPpkH2JJIAQXs1iMXcu1qyBRo3g22/N8HFuKwzSB0IIUfXyLHkknE5ge/J24hPj+fX4r3x+1+c0i2wGQOL5RBLOJBDiH0KvJr2IbR5LbPNYujXsRoBvQLn2lZ8Py5fDf/5jOkaDue6OH2+Ga61Xz9V/nXez1UDIV4OdsLA83n4bxo6FiRNNAPHvf5vz6oEH4LnnoFOnsrcT4Btg6zdRlNYafx9//Hz82JG8gx3JO3g//n0A6gbXZVLsJMZfMx4w/z++yleCinKQAEJ4La3h2Wfhyy/NPA8rVjhX/VlpJIAQQlSh0xmneWHVC2xP3s6uk7vIzs+2e3/DsQ22AOKZHs8w5qoxdGrQqdwBQ4EDB2DBAjNQxaFDZl1wsLlr/OKLNbeTdFmkCVPpWraEhQth61Z4+2344gtTIzF3LnTrZpokP/AAREaWb7tKKX58+EcyczOJT4xn3dF1rD2ylg3HNnAm8wyh/qG2tPN/m8/T3z9N5waduaL+FWaJMo/ONuOraSSAEF5Ja3jlFTOetL8/LFlimi+5lQzjKoRwkczcTPae3UvC6QT2nNlDwpkEEs4kEB0azdIHTHuhsIAw5myfQ742154WkS3o1KATnaM7071xd3o07mHbXlkdoEuyfz8sW2YCh19/LVzfsiWMGwePPAJ16pScXyBNmJzUpQt8/rnpC/Hee2ZUxfh4szzzjJnT6c474fbby1fLFewfTJ9mfejTrA9gaiaOph0lIrBwhtndp3ZzNvMsPx78kR8P/miXv3Wd1uwZv8dWOxF3KI4GYQ1oEdmCQL/AS//DvZQEEMLraA1//jO8+abpbjBnDvTr5+5SITUQQginZeVlcST1CIfOHeLwucPc1PommtYyDb9fXv0yr/78KtrBxAHRoYW3+YP8gvh08Kc0i2xGx+iOdj+IKurMGfjlF/jf/0yt7t69he+FhJj+Zg8+CDfdZD/8piiZNGEqnxYtzMhMb75pbg5+8gn88IMJZJctM9/73bub2a3794cePcykdc5SStn+1wq8fv3rjLt6HDuSd7Dr1C6znNzF7lO7qRVYyxY8WLSFWz67hay8LBRmO63qtKJ17da0qtOKgW0HVjhY9zYSQAivYrGYIQKnTjUT1MybB/fd5+5SWckwrkLUeNl52SRfSObE+RMAdG/cHYDz2ed54KsHSEpP4ljaMZIvJNvl+2LIF7YfNZFBkfj6+NKydkva1W1nlnrmsW3dtnb5hnUcVuGypqXBjh2wfbvpDP3LL7B7t32aWrXMnd+77jJ3fkNDHW9LFPPWW3D11dCvH9pSQhOm1avNpAgvvFDlxfMGwcGm6dIDD5hhgZcuNTNb//gjrF9vltdeM7Nad+xoDne3bmZp3978RnCWUoomtZrQpFYTbmt7m229RVs4m3nW9jo9J50+Tfuw7+w+Dqceti0FtRYxYTG2AOL9Te/z6s+v0iSiCY0jGtMkogkNwxsSFRpFg7AG3NLmFtccKDeRAEJ4jZwc09Z21ixzwVi40NwN8xjWJkwyjKsQ3k1rTUZuBqnZqaRlp5GWnca5rHOcyTjDmcwz3HvFvUSFRgHw9rq3+WLXF5zJPMPJ8yfJ+CnDtp1rGl3Dr4+adj8h/iGs2LcCizY3Gvx8/GgS0YRmkc1oVquZ3URYj3V7jPHXjMff1/+S/5a8PDhyxDRFKlj27TOBw4EDF6cPDDR3d/v0gZtvNsOxlueHmLC6+mq4917zRZVv/RyLfjWsXl34vihTTAw89phZ0tLg559NIPHjjyYALmjqVCAoCNq0gbZtoV07s7RtC82amb46zt7n81E+1AspbC8VERjByuFm5ICc/BwOnzvM/pT97Du7j/1n99Mlpost7ZHUIySlJ5GUnsSmxE12240OjSbp+STb6w7vdyDlfArN9jcjKjSK+iH1qR1cm9pBtenfor/tRkRqVirJF5KpHVSbyKBIl1wjKkouC8IrJCfD3XfDunXmwrBkialC9yhSAyGEQ0qpOsBM4EbgNPBnrfXnDtIp4J/Ao9ZVM4CJWuuL2/IUkWvJ5UDKATJyM+yWzNxMBl02CD8f81X36bZP2XNmDxm5GVzIvWALDtKy0+jfoj+v9HsFgJ0nd9Lxw44l7q9zg862ACLxfCKbT2y2veerfIkOiyYmLIYO9TsUrvfxZdkDy6gbXJdGEY2ICYvB18dxG6DSJsHKzzc/oM6dK1xSUsw18sQJ+yUpyawv6J5VXEAAXHGFmXSzUycz0VfXriaIEJeoXz8THNx7LzpnIaBosnA++KSbL7EHHjDve0T7W+8SEWGGgR040LxOTTUdsAuCiPh4Eyj/9ptZivPzM6M2NmkCjRubgKJ+fdOvol49++cREeb/wdF9wQDfANrUbUObum0clvO1/q/xp6v/xLG0YxxNO8rR1KMkpSdxMuMkIX6F/+Naa/ad3Ud2fjaJxxIv2s6bA960BRDf7/+e+74sbHYRFhBGeEA4YQFhhAWE8dOInwgPDAfgnV/e4UjqEcIDC98PCwgj2C+YVnVa0a2hmaMlOy+bg+cOEuwXTLB/sFOfAUgAIbzAli0weDAcPWr+6RcvNjd3PI70gRCiJNOAHCAa6Ax8q5TarrXeVSzdGGAw0AnQwCrgIPBhaRvfkbSDVu+1BRRoZX30Aa049mwitQIjsVjgkw1f8fOhn+3eL8hTK68dJy4zfawyztUmML014QERhAWEm0f/CCIDaxMZWJvzJxqQkGnS3hj+LF2vHUWYbyT7dh3mms49yc9X5Oaau/9LlphH8/pWkvNgu/W9rCwz+WVGhnksWIq/Tk8vDBbS0sp/8Bs1glat7JcOHcxdWX/33cCs/vr1g/nz0TdkAcE0+2wufDWncBxcCR5colYtiI01S4G0NNizxywJCWbZu9f8jjh1Cg4fNosz/PwgPNzxEhpqAoyCJSio6Gs/AgObEBTUhMDAHjQPhDb+4Bdg+g/973/m0ccHlvQ8xYZta2ncNoZz2WdIzTlLel4q53PO0STvOg4eNGlTT4XRzOdaUnPOkZp9jvT0fNJVLnAW1BkupAWS52cCngWbVxCfuBGUBrTd49Arh/LJ4JkoBbtP7aXrR53N+8pSxtEoclyc/4iEqFoWC3zwATz/vPmi7dEDvvrKVGV6JAkghLiIUioUuBvooLVOB9YqpZYCw4GJxZI/DLyjtT5mzfsOMJoyAog2J9ryn1enO3xv1+vbbc//ynP8leeYRQsWYPob9OYUL7GbddSj4WMmXTR1WcqMUvZ4nMMcB0yz9mhgHYpX6GXNn8kcNnKWIB6iuy3Xl/xCOLml/Sl2kkvIP5IO+JJOJOf4Eym0RKPQ1hYyFz/nOGb52WynJ/fgj4lEdvAGKXTlSv5CHUxNykEe4Qj3O11OwGH+lswCzI/kU/RmNy+Va5stmEVTFtjlr8c6rsDUFGUSzUbmlGubjvIHkUx3HrKl+YUvySXc6W2WlL/gOIfzHhYCUeRBlvXzv+GGcpW7OoqtxG1HAN2sS3FZBHKMxhylCcdozCnqc4r6nKae7bFgSSOC3LwAUlJMTV/lUEA44LhPxEy7V7daF8di3ir66n8lpvvcuhgdgLxi5SmbBBDCIx09asZ+/p/1/B81CqZN8/CqdRnGVQhH2gJ5Wus9RdZtB65zkPYK63tF013haKNKqTGYGgva0pYAByMWlSSUDGpzFoUmgjQC0ISQRTRJKDRRZJdrewD1OMu1rMefXOqSRwA+RHCBQSzBjzz8ySWUKALKMRlAY06wht6EcoHapHCUKeQTwUFa2gKA7bxFCldT0pd+WX+FxhdNgF3+wnXlcXF+bfe3+pR7m47y62LH79K2afJbsK+GseBXru2Wlb8Lz5SrjKJyBZFNa/bTmv1Opc8mgPOEO1wyCCGbQLIIIpvAi5bi63MIIB9fh0sefk69Z8HHdpvA9YvzN0AlgBAeJT/fdJJ+/nnTrrFuXfjwQxgyxN0lc4LUQAjhSBhQvOFNKji8xRtmfa9oujCllCreD0JrPR2YDtDtqm667y/2s9CWpq8vzPAz/6c6vw46rwV9feBFf5+CbaNzmpa2iYu36QNXrvuZ2NhYa35T3EGBhdcDS7bzzQMK+AQOtj1vas2vAs7ZblRcmWMpO0ooxlF+5b8RfMy65nkWmpfQb6LEbTrI//NaaHb9PADq5Wv65pWzoL59wa9Yfp++4P93AIK0pm9OObdZUv7AwhqXnhX4nBzlVwHn4Ntv4Z57TDV6gaAgM+1yQSP+GiouLo7You2OPFSgdansCdY95Xg4ew9UAgjhMdasgaeeMp2hwAwZOH06NGjg3nI5TTpRC+FIOqZFQVERwHkn0kYA6WV1okaBT2DF/u+Ur0L52n9jKqVQgRWvSSwpf0XLWFp+n4BL3KaD/D5+Ppf068CWv8jNfkfHuTy89nMKCjJ3xoKC0NnZqMBA22shvJn80hFu99tv5gZN374meGjc2MxG+fXXXhQ8gAzjKoRjewA/pVTRoUo6AcU7UGNd18mJdEJ4vtWrzWhLy5fDf//LoUceMTUPy5eb9atXu7uEQlSY1EAIt/n1V3j9dTM5DJhJY154wSwhJY9i6LmkBkKIi2itLyilFgGvKKUexYzCNAjo6SD5HOBZpdRyTMOc54CpVVZYIVyl6DwP1tGWDoeF0aKgiYp1iFcZylV4KwkgRJXKyDDXy+nTzSySYGpyR4+GCRPMuMxeS/pACFGSccAnwEngDDBWa71LKdUHWKG1DrOm+whoCRSM3j7Duk4I77JpU+nBQcE8EZs2SQAhvJIEEKLS5eXBTz+Za+WCBYXjmEdEwLhx8PTTZiIXrycBhBAOaa3PYuZ3KL5+DabjdMFrDbxgXYTwXi84cQr36yfBg/BaEkCISpGaampwly83EymdOlX43rXXwpgxpvY2NNR9ZXQ5GcZVCCGEEDWABBDCJc6ehQ0b4JdfTODw66+Fv6cB2rQxHaXvvx+uvNJ95axUUgMhhBBCiBpAAghRLhaLmf59xw4zetKOHbBhwzUcPWqfzs8Pevc2k20OHmyChmp/Y146UQshhBCiBnBJAKGUGg+MAK4E5mutR5SR/hngRSAE+BLToS7bFWURlyY/3zQ3SkqC48fh4EE4cKDwcf9+uHCheK4QAgPh6quhZ0/o1QtiY00fhxpFhnEVQgghRA3gqhqIROA14CYguLSESqmbgIlAf2u+xcD/WdeJS6Q1ZGdDZqaZ+DI9Hc6dM30SUlMLnxd9TE42AUNSkgkeLGVMwtmggalR6NjRPGZnxzNiRDcCAqrmb/RYUgMhhBBCiBrAJQGE1noRgFKqG9C4jOQPAzO11ruseV4FPsOJACLl2AX+O+FXtFbW/ZqBwu1ea9DYvwazzpnXJW63yOvjiVnsmLfd+p6yvW/32sF+LFqRb1Hk5SvyLIq8fJ/C19Z1+db1ttdF0ublK3LyfMjK9SEzx5esHF8yc6zPc31t6y5V/YhsGtTOokFkFi2iMmgZfYGWDS7QMjqDFlEXqBOea5d+586dBHx7tISt1SB79gDSB0IIIYQQ1Zs7+kBcAXxd5PV2IFopVVdrfaa0jAeSQ7n3X90rtXDO6VR2EjcKIJsgsggmk1AuEMk5apFa4mM0yTQgiQYkUZ9T+KflQRpw2Ln9dajUv8b7WPyka5EQQgghqi93/NIJA1KLvC54Ho6ZYMiOUmoMMAYg3KcN19WOQ6FRytziV9b6BmWtAzDrC+ogCt4vfM8urXU7jtKitG3b9mk12qLx8VElbsda71CkjCaNj7Lgp/LxJd88Kgu+quB5vu25H/m2tIXvFbzOI9gnmyCfbAJ9cgjyySa4yPNAn1x8VRltkBwKBVpxjlblzpmfl4ev/GgGIC8sjMPdupEYF+fuoniM9PR04uR42MjxEEII4e3K/NWnlIoDrivh7XVa697l3Gc6ULR7bcHz844Sa62nA9MBunXrppfFx5Zzd64XFxdHbMF09EKORzEJcjzsyPlhT46HEEIIb1dmAKG1jnXxPndh2gAttL7uBCSX1XxJCCGEEEII4X4u6e2plPJTSgUBvoCvUipIKVVScDIHGKWUaq+UigT+Bsx2RTmEEEIIIYQQlctVw8X8DcjEjKT0oPX53wCUUk2VUulKqaYAWuvvgLeA1cARTFfdl11UDiGEEEIIIUQlctUwrpOASSW8dwTTcbrouneBd12xbyGEEEIIIUTVkQHrhRBCCCGEEE6TAEIIIYQQQgjhNAkghBBCCCGEEE6TAEIIIYQQQgjhNAkghBBCCCGEEE6TAEIIIYQQQgjhNAkghBBCCCGEEE6TAEIIIYQQQgjhNAkghBBCCCGEEE6TAEIIIYQQQgjhNAkghBBCVAqlVB2l1GKl1AWl1GGl1NBS0k5QSu1USp1XSh1USk2oyrIKIYRwnp+7CyCEEKLamgbkANFAZ+BbpdR2rfUuB2kV8BCwA2gFrFRKHdVaL6iy0gohhHCK1EAIIYRwOaVUKHA38JLWOl1rvRZYCgx3lF5r/ZbWeovWOk9rnQB8DfSquhILIYRwllfVQGzevPm0Uuqwu8sB1ANOu7sQHkSOhz05HvbkeNjzlOPRrJK33xbI01rvKbJuO3BdWRmVUgroA3xUSpoxwBjry3SlVMIllNVVPOWz9RRyPOzJ8bAnx8OepxwPp74bvCqA0FrXd3cZAJRS8Vrrbu4uh6eQ42FPjoc9OR72atDxCAPSiq1LBcKdyDsJU0M+q6QEWuvpwPSKFq4y1KDP1ilyPOzJ8bAnx8Oetx0PacIkhBCi3JRScUopXcKyFkgHIopliwDOl7Hd8Zi+ELdprbMrp/RCCCEuhVfVQAghhPAMWuvY0t639oHwU0q10Vrvta7uBDjqQF2QZyQwEeirtT7mqrIKIYRwLamBqBiPqjb3AHI87MnxsCfHw16NOB5a6wvAIuAVpVSoUqoXMAiY6yi9UmoY8Dpwg9b6QNWV1KVqxGdbDnI87MnxsCfHw55XHQ+ltXZ3GYQQQlRDSqk6wCfADcAZYKLW+nPre32AFVrrMOvrg0BjoGizpXla68erttRCCCHKIgGEEEIIIYQQwmnShEkIIYQQQgjhNAkghBBCCCGEEE6TAMIFlFJtlFJZSql57i6LuyilApVSM5VSh5VS55VS25RSt7i7XFVJKVVHKbVYKXXBehyGurtM7iLnQ8nkelEzyOcs14EC8t1QSM6JknnbNUMCCNeYBmxydyHczA84ipllthbwN2ChUqq5G8tU1aYBOUA0MAz4QCl1hXuL5DZyPpRMrhc1g3zOch0oIN8NheScKJlXXTMkgLhESqn7gXPAD+4uiztprS9orSdprQ9prS1a62+Ag8BV7i5bVbCOeX838JLWOl1rvRZYCgx3b8nco6afDyWR60XNIJ+zIdcB+W4oTs4Jx7zxmiEBxCVQSkUArwDPurssnkYpFQ20pZRJo6qZtkCe1npPkXXbgZp6l8lODTwfLiLXi5pBPueS1dDrgHw3lKKGnhN2vPWaIQHEpXkVmCkzptpTSvkDnwGfaq3/cHd5qkgYkFZsXSoQ7oayeJQaej44IteLmkE+Zwdq8HVAvhtKUIPPieK88pohAUQJlFJxSildwrJWKdUZGAC85+6yVoWyjkeRdD6YmWZzgPFuK3DVSwciiq2LAM67oSweowafD3Zq2vWiupLvBXvyveAU+W5woIafEzbefM3wc3cBPJXWOra095VSTwPNgSNKKTB3GXyVUu211l0rvYBVrKzjAaDMgZiJ6Sh2q9Y6t7LL5UH2AH5KqTZa673WdZ2o2dWyNfl8KC6WGnS9qK7ke8GefC84Rb4bipFzwk4sXnrNkJmoK0gpFYL9XYXnMSfBWK31KbcUys2UUh8CnYEBWut0d5enqimlFgAaeBRzHJYDPbXWNfKLoqafD0XJ9aJmkM/5YnIdkO+G4uScKOTN1wypgaggrXUGkFHwWimVDmR5+gdeWZRSzYDHgGwgyRpJAzymtf7MbQWrWuOAT4CTwBnMBaCmfkHI+VCEXC9qBvmc7cl1wEa+G6zknLDnzdcMqYEQQgghhBBCOE06UQshhBBCCCGcJgGEEEIIIYQQwmkSQAghhBBCCCGcJgGEEEIIIYQQwmkSQAghhBBCCCGcJgGEEEIIIYQQwmkSQAghhBBCCCGcJgGEEEIIIYQQwmn/D/eNuANtmIyMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(z, np.sign(z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(z, logit(z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Activation functions\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(0, 0, \"ro\", markersize=5)\n",
    "plt.plot(0, 0, \"rx\", markersize=10)\n",
    "plt.plot(z, derivative(logit, z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "#plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Derivatives\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"activation_functions_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heaviside(z):\n",
    "    return (z >= 0).astype(z.dtype)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def mlp_xor(x1, x2, activation=heaviside):\n",
    "    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAEMCAYAAACfoCGmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcXHWZ7/HPk3S6sxFISEwMkATDEsRrkEUZEUUdxu0yhuXeURBc0LgML1fM5XpFkXEUM+I4IKKMLGMUrldZ3MaFGfUOyHhFuKhwDWggYZsEJE2SztKd7n7uH+cUVirV3VWnfmet7/v1qlfSVafOeaq6+lff+tV5zjF3R0RERESKZVLeBYiIiIjI3hTSRERERApIIU1ERESkgBTSRERERApIIU1ERESkgBTSRERERApIIS0DZrbezM7PYDsXmdm9GWxnkpl92cyeMjM3s5PS3uYE9VxnZt/LYbtL4sd/bNbbblJLy8+BmZ0U1z13nGXOMDMdn0fGpbEt9XpyGdsmUqS6WqnFzL5nZtdlVFJQpuOk7cnMjgbuBH7h7ie0ed+LgDPc/XkN188Dtrv7jkA1LgEeAo5z91/VXT8T6HP3p0JsZ5zt/2fgJuAk4EFgs7sPpbnNeLsnAT8F5rn7H+uu35fotfx02jU01LOEJr+HPLTzHJhZLzAH2ORjDABmdgbwTXe3sJVKXjS2tbR9jW0tKFJdrdQSh7g/uvtbMisskJ68CyigtwNfBM4xsyPc/XedrtDdn+y8rJa2MwAMZLCpQ4D/cPc7MtjWhNx9S9415K2d5yB+09mYYjlSTBrbJqaxrQVFqqtItaTC3XWJL8A04GngPwFXA59tssxC4OvAU8AO4B7g5cBbAG+4vCW+z3rg/Pj/1wM3NqxzEvAI8MH451cDtwH9wGbgR8ARdcs3budn8fUXAfc2rPfCeN2DwG+B19fdviS+/+nArfHj+X/AyeM8R9c1bHt9fP3PgC80WfZ7dT//jOhN4lPAH4EngM8Ck+qW6Y1v3xDX/CDw3rpa6y/XjbGdPuDzwCZgF/AL4CV1t58U3/+VwP+JH/evgKPbfL209PwBzwW+D2yLH/MNwIK6248Dfhw/J1uB24E/q7u9lddM43Pw0vhxDwBbgF8Cz2t4/HPrlj8nfs53AN8D/hrwhm2eAtwVP6cPAX8L9Ob9d6tLS69VjW0a29p9zXysrtaNwFfHefwzgK8SjTebgP9ONI5cV7fM+nid1xGNhY8AfwXsB/zP+L6/B/6ioY6Xxo9lV7zuv6du3GlSy/T4ulotH2mspUyX3Aso0gU4G/h1/P+T4j+0KQ0vxN8DPwdOBJYCpxENZNPiP8q1wIL4Mq3uxVkbyF4bv9j2rVvvy4Fh4Nnxz6fHl0OB5wP/C/hD7YVJ9KbuwKvi7cyJr7+IPQeyDxC96Z8JHAZcDIwAR8W3L4nXs5boDfhQ4J+IBumZYzxH+wKfiP/AFhBNz0PrA9mWuI7DgP8aP+431i1zA/Bo/PifEz835wCT4+faiULPgtpz2GQ7/wD8B/A64AjgH+M/2GfX/W6dKLi8HFhG9GbxO+JdAOLlHLhonNfLhM8f8GyiQfszcS3PB75LNOhMipd5BdFr74i4li8QvYnt38Zr5pnngGiGvJ/o9bg0XueZxG+GNIQ04EXAKPA/4t/LO+PH4HXbexXRa+mt8TpfDtxPkzd7XYp3QWObxrb2xrbT4+f3dcAi4FjgvHEe/5eIAt3JwJFEoWsLe4e0zcB74t/HpUSvl3+On4dDiD5APAFMje9zALA9Xv8RwH8mCoyXjlPLF4HHiF5DzwO+GT+W68Z6vEW+5F5AkS7xH1ptwLH4RXVG3e3vIPoEMHeM+19E3UBSd/36uvX2EKX7c+tu/wrw43HqmkE0AL0k/nlJ/Ed27Hjbj1+oH2vyGL/WsJ531t1+QHzdS8ap53ziT5kN621lIPv3hmVuBb4S///QeNuvHmO7J9EwA9S4nfi5GgLOqbt9MrAO+GTDel5Vt8wJ8XUH1l23lrqBqUk9Ez5/RIP2vzbcb3a8zAvHWK8RDcRvavU10/AczInX/7JWnkeiGZBbG5b5CnuGtH8DLmxYZgXRG4Q1244uxbmgsW2vv80x6tHYFt3+QaIPYVPGuL2+rplxXW9o+L32s3dIu6Hu55lxXZfVXbfH759otv737Dkj+Rai2b3pY9QyCJzVsJ2nKWlIU3dnzMwOAV5C9IaFR7/drwPn1i32AuA3XrdjZ7vcfRj4BnBWvN0+ok8tX6urZamZXW9m68xsK9HAN4noE02rj2cW0dcXP2+46XaiT2v1flP3/8fjf5/V6rba9JuGnx+v29YLiGZ0ftrB+pcCU6h73O4+Avw7bT5ud1/m7l9oYZvjrecY4KVmNlC7EH1Sr9WKmT0r7ih7wMy2EL1ZPov4993Ka6aeu28mGrh+ZGbfN7MPmtl4r50jiJ6feo0/HwP8j4bHcT3RYLxgnHVLzjS2PUNj25/uN9HY9k1gKvCQmV1tZv8l/n2OV9cv69a/HWjWjfubumUGiL6O/W3d7Zsaaj2CqNFltG6Z24m+Oj5kjFp6qRu/4u38tsmypaDGgT95O9GnkofNnmloMwAzO8jdHxnrjgl8Dfh3MzuA6KumXqKOoprvEU2Lv5PoE+Mw0f4UvYG27w0/737mBnePH3+7AX6U+PmqM6XJcrsbfvYE20pqzMddd1uSWsZ7/iYR7Y/W7DAFtQHpn4D5RF/hrCf6JPiv7Pn7nug1swd3f6uZfZ5oH6C/BP7WzFa4+4/afXB1j+MTRIN3o0x2HpfENLahsa2dWtz9ETM7nGjftj8n+mry42b2ojiAJdXsOUpaa+NjriTNpAFm1gO8mWhnx6PqLsuJkv9b40X/L/D8cY4vNUQ0GI7L3X9JtB/GG4k+dX47TvuY2f5E+xF8yt3/xaMOrH3YM1DXWsLH3Ja7byX6BNXYav8SokExtCeJ9r+qt7zNddxD9Jp8+Ri3T/i4iab+h6h73GY2Gfgz0nncE7mbaB+NDe7+h4bLtniZlwCXu/v33f0+opm0PZ7L8V4zY3H3X7v7Z9z9JKKvY948xqK/A45vuK7x57uBZU0ewx/iGRQpII1tQXTl2Obuu+Ix6QNE+woeyd7Pea2u3fEytbqmE+0P1qnfAcebWX1WeQnR87BunFqeGb/MbEagWnKhkBZ5HTAX+Ed3v7f+QrQD5Fst+gh2PdFOjd82sxPN7Dlm9pdmVvvDWw8sNrOjzWzuONPDEH3d8PZ42/VfW/UT7Wj+DjM7xMxeRrTTZP0b4RPATuBVZjY/Pk5MM38HnG9mbzSzw8zsYqKdgj/b6hPThp8Ar4mfj8PN7HPAQe2swN0fINqR+CtmdrqZHRw/z2fHi2wg+vT0OjObFx87qXEd24Ergc+Y2WvN7Ij45/lEO5S2zMzWmtl57dyniSuIdkj+hpm9KH7N/LmZXWVm+8TLPAC8ycyea2bHEb3mmh2baazXTGPdB5vZJWb2YjNbHL8+n8/YA/llwJ+b2X83s0PN7B3AqQ3LXAycaWYXm9nzzGyZRQe8Xd3i8yD50NjWua4b28zsLWb2djP7T2Z2MFGY3020f1hjXQPANXFdrzSz5xLtiziJzme7vkj01fYXzewIM3sdcAnRPoJ7HZsvruXquJaTzezIuLYJP2AUlUJa5Fzgp978QInfJNqZ8eT4j+RlRNP13yX6zv0T/OmFeCNRp8q/En36euM42/wacDhRB8yPa1fG373/FdGb6r1Eb/IXEn0FVltmmKh1++1Enyi/PcY2LiMazFbH6zoVON3dfz1OXUldU3f5OdFs0M0J1nMO0RvGZUQ7t15HFHJw98eAjxPtTLqJqAuymf9GtG/MtUSfYJ9PtMPuf7RZy+FEb3CJuXvtE/8o8EPgPqLf6SB/+p2+jWjn1ruI3jivIXpTbNT0NdPEDqIOs28SBcB/Inrj/MwYNf6C6G/g3USzK6cR7ahdv8yPiN50X06078kvgQuAh8epQ/Knsa1z3Ti2PU302rmN6Pk9HTjN3R8aY/nz42W/Q7Tf3W+IDv2xq8269hA/L68h2qfvHqLfwQ1Eh9UYy/lxDTfH/95L1PhUSjrjgIiIiAQTz7RuAP7O3S/Nu54yU+OAiIiIJGZmLyDqxPwl0X6G/y3+9xt51lUFwb7uNLPzzOxXZjZo45zI1MzebGZ3mdlWM3vUzFbHO7eKiORGY5hIRz5I1IDyE6L95F7q7o/mW1L5hdwn7XHgk0TfGY9nOvB+ou/DX0TU4tvs8AQiIlnSGCaSgLv/X3c/1t33cffZ7v5yd78r77qqINinP3e/CcDMjgUOHGe5K+t+fMzMvs7YbckiIpnQGCYiRVOEKfqXEnW8NWVmK4GVAFOn9h1z4EFpHSy6fe49mBXrEFGhahrxSYwMT8ZGRideeByTeyYxMtzZOkJSPeMrWj0AGx5Z/0d3n5d3HeMYcwxrHL8OKtD4BdmNYcMtfmkzySczaiMpVzOxEY/q7XFj2PJprvMmf4Y9NonhZjcE2WDj8XpbM2WSsXu09eco7aezZ5IxnMEYZi02XXY6fuUa0szsbUQnbn37WMu4+1XAVQCHHrbIv/UvxelGfXztB1i4rFiHiQpX0whr+l/Erdccz8Ibx+q6nthp7z+Om1bfGaCeMFTP+IpWD8AGLt+Qdw1jmWgMqx+/DjtskX/3X4v1oW792vNZsuySTLb1ra1HT7jM0kdWsO6gWzKoZmK3blzGmTuO4vrp9+RWw/pH93xv/8DMxVw6kO6fQ9/D7Z384b2HHMBlf3is7e3ssyGd9/JzTzyAq297jH3XDU68cId61068y90GvtjRLyy346SZ2Qrg08BrOjlfnKTn7Nl3cPLbfsH9qxbnXYpI4WgMa88Zs+7Ou4S2nLxgLbOmdHSYr44tOTD7M64NLmp2HO3wti1ONnPXqi1L+9iydLxjLnduaNmYe0UEk0tIM7NXA/8InOLupT3xaTc4e/Yd3Lzi89y/ajE7n3dA3uWIFILGsGTKFtQgCmt5WnLgk5mHtaoENaD0QS3kITh6zGwq0ekXJpvZ1GZt6Wb2CqKjn58en+dNSuDmFZ9n819vV1CTytIYlg0FtWQU1JIrc1ALOZP2UaJzrl0AvCn+/0fNbJGZDZjZoni5C4lOhfHP8fUDZvaDgHVIStYsv5bNf72dx08/OO9SRNKgMSwjCmrJ9PVmu0+jglrr0gpqwUKau1/k7tZwucjdH3b3me7+cLzcy929J76udnlNqDokXWuWX8vJb/uFgppUjsawbJ0x6+7ShbUiBDXNqCWXRVALHdZ0gnVpW62hQEFNRDqloNa+PIJaFmFt22LLpKEgbSGDmkKaJKLOTxEJpYxBLe+wps7P5MrU+amQJomp81NEQilbUIP8Z9XU+dmZMgQ1hTTpmDo/RSSE2ZN35F1C2/IOaqD91DqRxdefnVBIkyDU+SkiIWhGLRkFteSKHNQU0iQYdX6KSAgKaskoqCVX1KCmkCZBqfNTRELQITqSyTqojfaOqvMzRQppEpw6P0UklDIGtbzDmjo/k8ui87MdCmmSilrn5+CCXjUUiEhHyhbUIP9ZNXV+dqYoQU0hTVK1dL8n1PkpIh1TUEtGQS25IgQ1hTRJnTo/RSQEBbVkFNSSyzuoKaRJJtT5KSIhKKglo6CWXJ5BTSFNMqPOTxEJQZ2fyeicn8nlFdQU0iRT6vwUkVDKGNTyDmvq/Ewuj87Pnky3ViGjwyP09vwDo8MjTOqZnHc5pXL27Ds4e8UdnMr7WfTPw0y797G8S6q8n5x1NUPTJz7lzg/7gXdMvL7eHdN5xdfP7bwwyc3I8AhTJl/GyPAIk0s8hp0x626+tfXovMtoy8kL1nLrxmW5bb8W1NY/Oi+zbQ4uGqLv4d7E9183+nFG2Dbhcu9ZDxw08fp6RvZh+eMXJaply9I+9l03mOi+7dJMWkI7+7cyyR5kZ//WvEspLZ3zMzutBLQ81yfZG+gfwOxBBvoH8i6lY2WbUYPu/fozqVYCWjuGJ3e2vqxm1BTSEhgdHmFoYDtmztDAdkaHR/IuqbTU+SmSvZHhEXZs24GZs2PbDkYqMIYpqCVTpqBWNFkENYW0BHb2bwWPf3A0m9YhdX6KZGugf2CPMawKs2mgoJaUglpyaQc1hbQ21WbR6mk2rXPq/BTJRm0WrV5VZtNAnZ9JVbXzMwtpBjWFtDbtMYtWo9m0INT5KZK+PWbRaio0m1ZTxqCWd1ircudn2tLq/FRIa0OzWbQazaaFUTvn5/2rFquhQCSwZrNoNVWaTaspW1CD/GfVqnzOzyyEDmpBQ5qZnWdmvzKzQTO7boJlP2BmG81sq5ldY2b5nyRrAk1n0Wo0mxaUOj8la1Ufv2CMWbSaCs6mgYJaUgpqyYUMaqFn0h4HPglcM95CZvYq4ALglcBi4DnAJwLXEtR4s2g1mk0LS52fkrHKjl8w/ixaTRVn00BBLSkFteRCBbWgIc3db3L3W4CnJlj0zcDV7n6fu/cDfwO8JWQtoY07i1aj2bTg1PkpWany+AUTzKLVVHQ2DRTUklJQSy5EUMvrjANHAt+u+/nXwHwz29/d9xggzWwlsBJg3ry5PL72Y9lV+YwtTJ1yMdbCGScGtw6x5al3AbNSr6qZ3bvm8/jaVblsu5kQ9bwSOOrVM9h67EuY0t/ZUZ5nL5jBaauO62gdIWVVzw/7w68zq+fxh+/LZDPtSDx+rV97YXZV7mELvT1/09IYtn3Lbvqfeg95jGGDuxawfu0Fqa3/2Pjf/pHpLS3fN7QfSx9ZkVo9rVgKbN09FYA5o9M5c8dR2RcxBwaHmseF+ZP7+NDMwM1ez43+mTT0p3mk8x4IuwmAt77ywHg7E316Se6uqzu7f14hbSawpe7n2v/3oeFTrLtfBVwFcOhhi3zhstWZFFhv+5P9DG3b3dKyZrvZd/9PM2Pe7JSrau7xtavI4zkaS6h6FgJr+l/MTf/7hRy+ekPi9Zy26jhuWn1nx/WEklk9LZzqqV1Feh4zlmj8OuywRb5k2SWZFNhoy5Nb2LG19TFs9v6fZt95+6Zc1d7Wr72ALJ6jJdDSqaSWPrKCdQfdkno9LXvwDVw//Z58tj29+WmkPjRzMZcOJB+TJ9LJqaQmctkf/nRKwn02pBfUOpFXd+cAe35Mq/0/7HkfAmhlX7RG2jctHer8lIIozfgFre2L1qiq+6bVK+PXn7Om7Mp1+1Xu/Ez75OxJ5RXS7gOW1/28HNjU+FVBEbS0L1oj7ZuWKnV+Ss5KM35Bi/uiNarwvmn1yhjUtJ9aeooY1EIfgqPHzKYCk4HJZjbVzJp9pfpV4Fwze66Z7Qd8FLguZC0hJJlFq9FsWrrU+SmhVW38gmSzaDXdMJsGCmpJ5XHg2ywULaiFnkn7KLCTqD39TfH/P2pmi8xswMwWAbj7D4HVwE+Bh4ENwMcD19KxRLNoNZpNS506PyWwSo1fkHAWraZLZtNAQS0pBbX0BW0ccPeLgIvGuHlmw7KfAz4XcvshdTKLVjM0sJ1ps2cxqWdyoKqk0dmz74C3wa0cz8IbH8q7HCmxKo1f0NksWs2ObTuYOXsmk7tgDKsFtVYaCori5AVruXXjslxr6OsdznX7aakFtbwbCnRaqDF0NItWo9m0TOicnyJ762gWraaLZtNqyjar1q3n/MxK3rNqCmlNhJhFq9G+adlQ56fIn4SYRavpln3T6pUtqEH+X3/m0fmZlTyDmkJaE0Fm0Wo0m5YpdX6KBJpFq+nC2TRQUEtKQS0shbQmRgbDtvuGXp+MT52fe+vd0doR1vNan4Q1tCvsmBN6fWUxe3KY2cgsVTWoTZo0c+KF2jCZfdq+Tx5BLa8zDhTarAPnt7Rc0Y7uL3+yZvm1rFn0YjUUxF7x9XNbWq5oZ2SQZOYdtPeR4ZvJ6gj/ZXbGrLtL1UwAxWgoWHLgk03PUJDUQYs+OuEyfQ/38t5DDtjjTAKhbVtsmTYTaCZNKqvWUKAZNRHpxBmz7i7d159VnVEbz+CiIUZ7R1PfzrbFltmsmkKaVJo6P0UklDIGtbzDWh77qFXpDAUKaVJ59Z2fPm1K3uWISImVLahB/rNqOudncgpp0jVuXvF5hp81qs5PEemIgloyCmrtU0iTrnLwtKfU+SkiHVNQS0ZBrT0KadJ1dM5PEQlBQS0ZBbXWKaRJV1Lnp4iEoM7PZPIIalmEtdCdnwpp0rXU+SkioZQxqOUd1tT5OTGFNOlqOueniIRStqAG+c+qqfNzfAppIuicnyIShoJaMlUNap1SSBOJ6ZyfkqZhDbddQ0EtGQW1vWnUEKmjzk9JU9nOASnJKaglo6C2J4U0kQbq/JQ0Kah1D3V+JlPVzs8kFNJEmlDnp6RJQa27lDGo5R3Wqtz52Q6FNJExqPNT0qSg1l3KFtQg/1m1Knd+tkohTWQC6vyUtCiodRcFtWS6OagFDWlmNsfMbjaz7Wa2wczOHGO5PjP7kpltMrPNZvZdM9M7oBSWOj+rL6/x61tbj1ZY6yIKasl0a1ALPZN2BTAEzAfOAq40syObLPc+4M+A5wMLgX7g8sC1iASlzs/Ky3X8UlDrHgpqyXRjUAsW0sxsBnA6cKG7D7j77cB3gLObLH4w8CN33+Tuu4BvAM0GQ5FCUednNRVl/FJQ6x7q/Eym2zo/zd3DrMjsBcDP3X163XXnAy9z91Malj0W+AfgvwBPA18BnnD39zdZ70pgJcC8eXOP+aevfSxIvSHs3jWfKVM35V3GHopWU1XreWpkBk9vm0Hfxs7+eGcvmEH/xu0d1xNK0eoBWPm+c+5y92PT3EYW49fceXOPufyrn26pntmTdyR8JO0Z3LWAvqkbM9lWK7q1nv6R6RMvBPQN7cdg79MpV9OarbunAjBndDqbJ2Xzeq03ONTT9Pr5k/vYNDKYyjYnDbU/r3XemX/V0fjV/FEmMxPY2nDdFmCfJsv+HngEeAwYAX4LnNdspe5+FXAVwKGHLfKFy1aHqrdjj69dRZHqgeLVVNV6Fsb/nnrL+1n0z8NMu/exROs5bdVx3LT6zo7rCaVo9WQo9fHrOYct8XUH3dJyQVnMsqxfewFLll2S+nZa1a31LKG1WdSlj6ygnddQ2m7duIwzdxzF9dPvyX7jca5d/+i8Pa7+0MzFXDqwIbXN9j3cm9q6mwm5T9oAMKvhulnAtibLXgH0AfsDM4CbgB8ErEUkE+r8rIzCjV/66rO7lO2rT+jerz+zFDKkPQD0mNmhddctB+5rsuxRwHXuvtndB4l2un2hmc0NWI9IJtT5WQmFHL/U+dldyhjUZk3ZlXcJlQ5qwUKau28n+kR5sZnNMLMTgNcDa5osfidwjpnta2ZTgPcAj7v7H0PVI5IldX6WW9HHLwW17lHGoKYZtfSEPgTHe4BpwBPADcC73f0+MzvRzAbqljsf2EW0b8eTwGuBUwPXIpIpdX6WXqHHLwW17qHOz2Sq2PkZsnEAd98MrGhy/W1EO+bWfn6K6DhEIpVy9uw74G1w09IXcvjq9HZelfDKMH59a+vRpXvzluTOmHV3qcJ5LajdunFZbjX09Q5nvs3BRUOpNRTotFAigemcn5KmMr1pS+fKGMrznlWr0jk/FdJEUqLOT0mLglp3UVBLpgpBTSFNJEXq/JS0qPOzuyioJVP2oKaQJpIydX5KmhTUukdWZ6IISUGtMwppIhlQ56ekSUGte6jzM5mydn4qpIlkpBbU7l+1OO9SpIIU1LpLGYNa3mEt66AWgkKaSIbU+SlpUlDrLmULapD/rFoenZ+dUEgTyUGt89OnTcm7FKkYBbXuoqCWTFmCmkKaSE7WLL+W4WeNaj81CU6dn91FQS2ZMgQ1hTSRHB087Sk1FEhqFNS6h4JaMkUPaqUKaYMj+mpIqkedn5ImBbXuoc7PZIoc1EoV0ibtHuXUW97Pmv4X512KSFDq/JQ0Kah1lzIGtbzDWlGDWqlCGsDhqzdw6zXHK6hJ5ajzU9KkoNZdyhbUIP9ZtSJ2fpYupAEsvPEhbr3meM7+9VvzLkUkOJ3zU9KioNZdFNSSKVJQK2VIgyiozblihoKaVJLO+SlpUednd1FQS6YoQa20IQ1g2r2PMeeKGZx6y/vzLkUkOJ3zU9KkoNY9FNSSKUJQK3VIgyioHb56g4KaVJI6PyVN/SPT8y5BMqLOz2TyDmqlD2k1taCmhgKpGnV+Spo0o9ZdyhjU8g5reQa1yoQ0UOenVJc6PyVNCmrdpWxBDfKfVcur87NSIQ3U+SnVps5PSYuCWndRUEsm66BWuZAG6vyUalPnp6RFnZ/dRUEtmSyDWtCQZmZzzOxmM9tuZhvM7Mxxlj3azP7NzAbMbJOZvS9kLer8lCpT52d4RRq/8qag1j0U1JLJKqiFnkm7AhgC5gNnAVea2ZGNC5nZXOCHwJeB/YFDgB8HrkWdn1Jp6vwMrlDjV94U1LqHOj+TySKoBQtpZjYDOB240N0H3P124DvA2U0W/yDwI3f/ursPuvs2d/9dqFoaqfNTqkqdn2FkMX6NePn2LlFQ6y5lDGp5h7W0g1rIUeMwYNjdH6i77tfAXp9EgeOBzWZ2h5k9YWbfNbNFAWvZizo/parU+RlEJuPXrRuXBSg1Wwpq3aVsQQ3yn1VLs/PT3D3MisxOBL7p7gvqrnsHcJa7n9Sw7APAs4CTgd8Cq4Fj3P2EJutdCawEmDt37jGfuvBzHdW5e3YfzBrm4GlPdbQegN275jNl6qaO1xNS0WpSPeMLXc9DO/en54lJ2M7die4/e8EM+jduD1ZPCCvfd85d7n5smtvIZPyaN/eYv73675+5bdaUXeEfSJv6hvZjsPfplpadPXlHytXA4K4F9E3dmPp2WtXN9bR6oON2XkNp27p7KnNGp7N5Uvqv1bEMDvXs8fN7z3hjR+NXz8SLtGwAmNVw3SxgW5NldwI3u/udAGb2CeCPZravu2+pX9DdrwKuAliy6GC/afWdHRe683kHsPmvt7Nm+bUdrefxtatYuGx1x/WEVLSaVM/4QtezEDj712/F/mU2C298qO37n7bqOEL8jZVQ6uPX4kOf49dPv2ePFeU9A7D0kRWsO+iWtu6T5kzL+rUXsGTZJamtv13dXM+XhkxWAAAb4ElEQVQSWptFTfIaStWDb6Dx7yxT02H9o/OCrS7k150PAD1mdmjddcuB+5os+xugfgovzHRei9T5KVWmzs9Echm/9PWnFFkZv/oswgx1yK8+g4U0d98O3ARcbGYzzOwE4PXAmiaLXwucamZHmdkU4ELg9sZPoWlS56dUmTo/25Pn+KWgJkWmzs9kQgW10O1G7wGmAU8ANwDvdvf7zOxEMxuoLeTuPwE+Anw/XvYQYMxjEqVJnZ9SVer8bFtu45eCmhRdGYNa3mEtRFALGtLcfbO7r3D3Ge6+yN2vj6+/zd1nNix7pbsf4O6z3f0Ud38kZC3tUOenVJU6P1uX9/iloCZFV7agBsWYVetE+Q7ckxKd81OqTOf8LIdbNy4rXVhTUOsuCmrZUkiro3N+SpXVzvmpoFZ8ZQxqCmvdQ0EtOwppDdT5KVW2Zvm1HPuZu9VQUAJlC2qgWbVuoqCWDYW0JtT5KVWmzs/yUFCTIjtj1t2ZHOQ4pLIFNYW0cajzU6pKnZ/loaAmRVe2WbUidH62SiFtAur8lKpS52d5KKhJ0ZUtqEE5ZtUU0lqgzk+pMnV+loM6P6XoFNTCU0hrkTo/pcpqnZ+7Z/flXYpMoIxBTWGteyiohaWQ1gZ1fkqVrVl+LbPmDqihoATKFtRAs2rdREEtHIW0NqnzU6ps/8nb1flZEgpqUmQ652cYCmkJHb56A+uefpYaCqRy1PlZHgpqUnRlDGpFCmsKaR3o2zikzk+pJHV+loeCmhRd2YIaFGdWTSGtQ+r8lCpT52c5qPNTik5BLRmFtADU+SlVpnN+lkcZg5rCWvdQUGufQlog6vyUKtM5P8ujbEENNKvWTRTU2qOQFpA6P6XKdM7P8lBQkyJT52frFNJSoHN+SlWp87M8yhjU+kem512CZKiMQS3rsKaQlhKd81OqSp2f5VHGoKYZte5StqAG2c6qKaSlSJ2fUmXq/CwHdX5K0SmojU0hLWXq/JQqU+dneZQxqCmsdQ8FteYU0jKgzk+pMnV+tsZH866gfEENNKvWTRTU9hY0pJnZHDO72cy2m9kGMztzguV7zex3ZvZoyDqKSJ2fUmVV6PzMYvxa/+i8zgvtkIKaFJk6P/cUeibtCmAImA+cBVxpZkeOs/yHgScD11Bo6vyUqqpA52cm45eCWjIKat2ljEEtjbAWLKSZ2QzgdOBCdx9w99uB7wBnj7H8wcCbgE+HqqEs1PkpVVXWzs+sx6/1j87LPawpqEnRlS2oQfhZtZAzaYcBw+7+QN11vwbG+iR6OfARYGfAGkpDnZ9SZSXs/Mxl/Mo7qG3dPbV0YU1Brbt0e1Azdw+zIrMTgW+6+4K6694BnOXuJzUseyqw0t1fY2YnAV9z9wPHWO9KYCXA3Llzj/nUhZ8LUm8IsxfMoH/j9o7W4dOmMPysUQ6e9lSQmnbvms+UqZuCrCsE1TO+qtfz0M796XliErZzd+J1rHzfOXe5+7HBimoik/Fr3txjPnbl5U2339c7HOJhtG3O6HQ2T9oBwKwpu3KpoV7f0H4M9j7d8vKzJ+9IsRoY3LWAvqkbU91GO4pWD2RXU6sHOm73NZSmrbun8s6/PLuj8asnYD0DwKyG62YB2+qviL9WWA28tpWVuvtVwFUASxYd7DetvrPzSgM5bdVxhKhn5/MO4OHX9nDzis93vK7H165i4bLVHa8nFNUzvqrXsxBY0/9ibr3meBbe+FCw9aYg9fFr0dLn+KUDG8ZcdsmB2e+ee+aOo7h++j3P/Jz3yaSXPrKCdQfd0tZ90pxpWb/2ApYsuyS19beraPVAdjUtobVZ1CSvoSIL+XXnA0CPmR1ad91y4L6G5Q4ler5vM7ONwE3As81so5ktCVhPaajzU6qsJJ2fuY9feX/1CdpPTYqtjJ2fnQoW0tx9O9GAdbGZzTCzE4DXA2saFr0XOAg4Kr68HdgU//+RUPWUkTo/paqK3vlZlPFLQS0ZBbXu0k1BLfQhON4DTAOeAG4A3u3u95nZiWY2AODuw+6+sXYBNgOj8c8jgespHXV+SlWVoPOzEOOXOj+TUVDrLt0S1IKGNHff7O4r3H2Guy9y9+vj629z95lj3OdnY+10263U+SlVVtTOz6KNX0UIamULawpq3aUbgppOC1VQOuenVJnO+dmavIMalG9WTef87C5VD2oKaQWmc35Klemcn61RUEtGQa17VDmoKaQVnDo/pcpK0vmZOwW1ZBTUukdVOz8V0kpCnZ9SVUXv/CwKBbVkFNS6S9oHOM6aQlqJqPNTqqoEnZ+FoM7PZBTUukuVZtQU0kpGnZ9SZUXt/CyaIgS1soU1BbXuUpWgppBWQur8lCpT52dr8g5qUL5ZNXV+dpcqBDWFtJJS56dUmTo/W6OgloyCWvcoe1BTSCsxdX5KldUaCmR8CmrJKKh1jzJ3fiqkVYA6P6Wqzp59R94llIKCWjIKat2ljEFNIa0iap2fT43MyLsUEcmBOj+TUVDrLmULagppFbLwxofY+seZaigQ6WJFCGplC2sKat2lTEFNIa1ipvQPqvNTpMvlHdSgfLNq6vzsLmUJagppFaTOT5ECcst0cwpqySiodY8yBDWFtIpS56dI8fQ93Jvp9hTUklFQ6x5F7/xUSKs4dX6KFIuCWjkoqHWXogY1hbQuoHN+ihRLHkEt77BWxqDWPzI97xIkQ0UMagppXULn/BQplqyDGuQ/q6bOTym6ogU1hbQuonN+ihRL38O9+vqzBNT52V2KFNQU0rqMOj9FikdBrRwU1LpHUYKaQloXUuenSPEoqJWDglr3KELnp0JaF1Pnp0ixZB3UBod6Mt1eMwpqUnR5BrWgIc3M5pjZzWa23cw2mNmZYyz3YTO718y2mdlDZvbhkHVI69T5KRIpyvilzs9yUFDrLnkFtdAzaVcAQ8B84CzgSjM7sslyBpwDzAZeDZxnZm8IXIu0SJ2fIkCBxq9u7fzcuntqrjW0S0Gtu+QR1IKFNDObAZwOXOjuA+5+O/Ad4OzGZd19tbvf7e7D7n4/8G3ghFC1SPvU+SndrIjjlzo/y0Gdn90l66Bm7h5mRWYvAH7u7tPrrjsfeJm7nzLO/Qy4G/iyu3+pye0rgZUAc+fOPeZTF34uSL0hzF4wg/6N2/MuYw+d1uTTpjC0r7F0vyeC1LN713ymTN0UZF0hqJ7xFa0egNe+6r13ufuxaW4jm/Fr3jEXXfaFRPWN9o4mut9E5k/uY9PI4F7X9/UOp7K9icwZnc7mSTsAmDVlVy411Osb2o/B3qdbXn725B0pVgODuxbQN3VjqttoV9FqyqqeVg90/MbXvKOj8SvkXqMzga0N120B9pngfhcRzehd2+xGd78KuApgyaKD/abVd3ZWZUCnrTqOItUD4Wq6f9Vibl7x+Y7X8/jaVSxctrrj9YSiesZXtHoylPr4teg5S/2yPzyWuMDBRUOJ7zuWD81czKUDG5retuTAJ4NvbyJn7jiK66ff88zPJy9Ym3kN9ZY+soJ1B93S1n3SnGlZv/YCliy7JLX1J1G0mrKqZ0n8b9qzqCH3SRsAZjVcNwvYNtYdzOw8on07Xufue3+ck9yo81O6TOHHL331WQ766rO7pP31Z8iQ9gDQY2aH1l23HLiv2cJm9jbgAuCV7v5owDokEHV+Shcpxfilzs9yUFDrLmkGtWAhzd23AzcBF5vZDDM7AXg9sKZxWTM7C/gUcLK7PxiqBglPnZ/SDco0fnVr52fZwpqCWndJK6iFPgTHe4BpwBPADcC73f0+MzvRzAbqlvsksD9wp5kNxJe9drqVYlDnp3SJ0oxf6vwsB3V+dpc0glrQkObum919hbvPcPdF7n59fP1t7j6zbrmD3X2Ku8+su7wrZC0Sls75KVVXxvFLQa0cFNS6R+igptNCSct0zk+R4lFQKwcFte4R8pyfCmnSNnV+ihSLglo5KKh1lxBBTSFNElHnp0ixqPOzHBTUpB0KaZKYOj9FikWdn+WgoCatUkiTjqjzU6Q15rDPhjCn4RuPOj/LQUFNWqGQJh1T56dI67IIaqD91MpAh+iQiSikSRDq/BRpnYJaesoW1ECzajI2hTQJSp2fIq1RUEuPgppUhUKaBKfOT5HWVDmo5R3WFNSkChTSJBW1zs+Hdu6fdykihVbVoAb5z6qp81PKTiFNUrPwxofoeWKSOj9FJrDPBlfnZ4oU1KSsFNIkVbZztzo/RVpU1Vk1BbX2qfNTQCFNMqDOT5HWKailp2xBDTSr1u0U0iQz6vwUaU1WQW3SULZvAQpqySioda+evAsoop+cdTVD03dMuNwP+4F3TLy+3h3TecXXz+28sAo4fPUGbl13PLwNzp59R97lSJ3R4RF6e/6B0eERJvVMzrucrrfPBmfbYkt033WjH2eEbRMud94Dra1v0qSZHLToo4lqaVQLaksOfDLI+pK4deMyTl6wNrftJ/GtrUdzbN5FFNzI8AhTJl/GyPAIkysyhmkmrYlWAlqe6ys7nfOzmHb2b2WSPcjO/q15lyKxpDNqrQS0doyODgRdH+Q/q1bGzs/+kel5l1BoA/0DmD3IQH/412teFNIkFzrnZ7GMDo8wNLAdM2doYDujwyN5lySxrDo/85B3UIPyff2prz6bGxkeYce2HZg5O7btYKQiY5hCmuRG5/wsjp39W6GWAxzNphWQglp6yhjUFNb2NNA/sMcYVpXZNIU0yZU6P/NXm0Wrp9m0YlJQS0/ZghpoVq2mNotWryqzaQppUgjq/MzPHrNoNZpNKywFtfQoqJXTHrNoNRWZTVNIk8LQOT+z12wWrUazacVV1aA2ONSTe1hTUCuXZrNoNVWYTQsa0sxsjpndbGbbzWyDmZ05xnJmZp8xs6fiy2fMLFmvuVSKOj+z1XQWrabLZtPKNn5VNahB/rNqZez87Nag1nQWraYCs2mhZ9KuAIaA+cBZwJVmdmST5VYCK4DlwPOBU4B3Bq5FSkqdn9kYbxatpstm00o3fmXZ+dmNZyjYuntq3iW0pduC2nizaDVln00LFtLMbAZwOnChuw+4++3Ad4Czmyz+ZuBSd3/U3R8DLgXeEqoWKT91fqZv3Fm0mi6ZTSv7+KWglp4yzqh1S1gbdxatpuSzaeYe5o/bzF4A/Nzdp9dddz7wMnc/pWHZLcBfuPv/iX8+Fvipu+/TZL0riT65Mnfu3GM+deHngtQ7npX95wRf51Wzvxp8nc3MXjCD/o3jz45kKUQ9gwt6WbrfE0Hq2b1rPlOmbgqyrhDyq2cLU6dcjNnuCZd0n8Ku3R8HZqVfVhOvfdV773L3VA+2ntX4dfFnv5Dio4DR3uhb1/Me+Kvg6/7CYd+o285o8PUDzJ/cx6aRwb2u7+sdTmV7E5kzOp3Nk6KZmllTduVSQ72+of0Y7H265eVnT07/QOqDuxbQN3Vj6tvZ2xZ6e/6m5TFsaPhj5DGGvfov3tfR+BXytFAzgcaP3FuAvQaueNktDcvNNDPzhtTo7lcBVwEsWXSw37T6znAVj6WFUz21K5O6gdNWHZfZtloRqp77Vy3mtJf9suNTST2+dhULl63uuJ5Q8qpn+5P9DG2beHADMNvNvvt/mhnzZqdcVa5SH78WL36OX33bY+EqHkPSU0lN5LI/7Fn74KKh4Nv40MzFXDqwoelteZxG6swdR3H99Hue+TnvU0ktfWQF6w66pa37nDHr7pSqiaxfewFLll2S6jaa2fLkFnZsbX0Mm73/p9l33r4pVxVeyH3SBtg7ps6CpucnaVx2FjDQOMCJ1KjzM5xW9kVr1AX7pmUyfu27bu9ZotCq/NVn3l9/lu2rT6jmfmqt7IvWqKz7poUMaQ8APWZ2aN11y4H7mix7X3zbRMuJPEOdn2G0tC9ao+rvm5bZ+LXvusFMwloWsg5qkP9+aur8zF9L+6I1Kum+acFCmrtvB24CLjazGWZ2AvB6YE2Txb8KfNDMDjCzhcCHgOtC1SLVpc7PziSZRaup8mxaHuNXlYKaGgqKrypBLcksWk0ZZ9NCH4LjPcA04AngBuDd7n6fmZ1oZvUR9svAd4HfAvcC34+vE5mQOj+TSzSLVlP92bTMx6+qBDVQ52cZVKHzM9EsWk0JZ9OChjR33+zuK9x9hrsvcvfr4+tvc/eZdcu5u69y9znxZZX2R5N26Jyf7etkFq2m4rNpuYxfCmrJKaglU9ag1sksWk3ZZtN0WigpNZ3zs3UdzaLVVH82LRcKaskpqCVTxqDW0SxaTclm0xTSpPTU+TmxELNoNVWeTcuTglpy6vxMpkxBLcQsWk2ZZtMU0qQS1Pk5viCzaDWaTUuNOj87U4SgVrawVpagFmQWraZEs2kKaU307pg+8UI5rk+aU+fn2EYGwx54NPT6ZE+dBrWekWbH4E1uctNj+k5MnZ/lUIagNrQr7JgTen1pCXnGgcp4xdfPbWm5oh3dX2qdnwdw6mvfz80rPp93OYUx68D5LS1XtDMydLN91w2yZWlfovsuf/yilpY798QD+PzDjyfaRjv6Hu5N5QwFY1n/6LxczlBQ79aNy3I/Q0E7akEt7TMUJDXvoNbCd15nQEiLZtKkctT5KVWhMxQkpxm1ZMowq9ZNFNKkstT5KVWgoJacgloyCmrFoZAmlabOT6kCBbXk1PmZjIJaMSikSeWp81OqIIvOz6oGNch/Vk2dn5KEQpp0hVrn50M798+7FJGOZBHUsghr6vwsBwW1fCmkSdeYdu9j9DwxSQ0FUnr6+jM5BbX2VeGcn2WlkCZdxXbuVuenVIKCWnIKaskoqGVPIU26kjo/pQoU1JJTUEtGQS1bCmnStdT5KVVQpaA2aSjbtyR1fiajoJYdhTTpaur8lCpQ52dnihDUyhbWFNSyoZAmXU/n/JQsmKcfctT5mVzeQQ3KN6umoJY+hTQRauf8nKGGAklV79pH6V37aKrbqNLXnwpqxdc/Ml1hLUUKaSIxnfNTsqKg1joFtXJQUEuHQppIA3V+ShYU1FqnoFYOCmrhKaSJNKHOT8mCglrr8ghqg0M9mW6zkYKaKKSJjEGdn5KFLIKaOj+Ty3tWTZ2f3S1ISDOzOWZ2s5ltN7MNZnbmOMt+2MzuNbNtZvaQmX04RA0iaVDnZ3fIewxLO6iBOj87kXdQg/LNqimohRFqJu0KYAiYD5wFXGlmR46xrAHnALOBVwPnmdkbAtUhEpw6P7tC7mOYOj/b041BbevuqXmX0Bad87NzHYc0M5sBnA5c6O4D7n478B3g7GbLu/tqd7/b3Yfd/X7g28AJndYhkiZ1flZX0cYwBbXWdWNQK9uMGmhWrRPmHR5g0cxeAPzc3afXXXc+8DJ3P2WC+xpwN/Bld//SGMusBFbGPz4PuLejgsOaC/wx7yIaFK0m1TM+1TOxw919n7RWnuYYVvDxC4r3+1Y94ytaPVC8mopWT0fjV4jWlZnA1obrtgCtFHUR0WzetWMt4O5XAVcBmNmv3P3YZGWGV7R6oHg1qZ7xqZ6JmdmvUt5EamNYkccvKF5Nqmd8RasHildTEevp5P4Tft1pZj8zMx/jcjswAMxquNssYNsE6z2PaL+O17l7+vPvItKVNIaJSFlNOJPm7ieNd3u8P0ePmR3q7r+Pr14O3DfOfd4GXAC81N3Tb2sSka6lMUxEyqrjxgF33w7cBFxsZjPM7ATg9cCaZsub2VnAp4CT3f3BNjd3VUfFhle0eqB4Name8ameiaVaU4ZjWNc9twmonvEVrR4oXk2VqqfjxgGIjjEEXAOcDDwFXODu18e3nQj8wN1nxj8/BBwI1H898DV3f1fHhYiIJKAxTESKKEhIExEREZGwdFooERERkQJSSBMREREpoEKHtLzPp9dODRb5jJk9FV8+Ex/oMqg26snkHKnt/I7i5XvN7HdmlkpHXJuvmaPN7N/MbMDMNpnZ+/Kqx8z6zOxLcR2bzey7ZnZACvWcZ2a/MrNBM7tugmU/YGYbzWyrmV1jZn2h62mnJjN7s5ndFdfzqJmtNrMQx3pMhcavjurJ7BzPGsPC1NOtY1ja41ehQxoFOJ9eGzWsBFYQte4/HzgFeGeA7SetJ6tzpLbzOwL4MPBkCnW0VY+ZzQV+CHwZ2B84BPhxXvUA7wP+jOi1sxDoBy5PoZ7HgU8S7SQ/JjN7FdEhJl4JLAaeA3wihXpargmYDryf6IjiL4prOz+lmkLQ+JW8nizP8awxLEA9dO8Ylu745e6FvAAziF4Yh9Vdtwa4pMX7XwZcnlUNwB3AyrqfzwV+UZTnJMTz0Wk9wMHA74DXAI/m+ZohOoTCmtA1dFDPlcDqup9fB9yfYm2fBK4b5/brgU/V/fxKYGPKz9e4NTVZ/oPAd9OsKYvf/Rj31/gV+PkIUZPGMI1hSetpsnxL41eRZ9IOA4bd/YG6634NjPcJB3jmfHonMs7BKFOo4cj4tomWy6qeZwR8Pjqt53LgI8DOwHUkqed4YLOZ3WFmT8RT84tyrOdq4AQzW2hm04k+sf4gcD3taPZ6nm9m++dUTzMvJfxrOhSNX53V84wUx68kNWkM0xgWSkvjV5FDWqrnBE2hhpnxbfXLzQy8X0fS5+Qiwjwfiesxs1OBye5+c+AaEtVDdJyrNxNN0S8CHgJuyLGe3wOPAI/F9zkCuDhwPe1o9nqG1v7+UmfREf+PBT6bdy1j0PjVWT31LiKd8autmjSGTViPxrAWtTN+5RbSrBzn02unhsZlZwEDHs9rBtL2cxL4+UhUj0Wn3VkNvDfw9hPVE9sJ3Ozud7r7LqJ9FV5sZvvmVM8VQB/RviUziI6An+en0GavZ5jg7y8LZrYC+DTwGnf/Y041aPxKtx4gk/OjagwLV4/GsBa0O37lFtLc/SR3tzEuLwEeID6fXt3dWj2f3is9zPn02qnhvvi2lmrNoJ40no+k9RwKLAFuM7ONRH+8z467bpbkUA/Ab4D6N6A0jurcTj1HEe3PsDl+M7oceGG8c3Aemr2eN7n7UznVA4CZvRr4R+AUd/9tXnVo/Eq9nizGr3Zq0hg2cT0awyaQaPxKaye6QDvi/U+i6dsZwAlE05VHjrHsWcBG4Ig8agDeRbRD6QFEnS33Ae/K6zlJ6/lIUg/QAyyou5xG1BGzgOjrgzyen1cQdR8dBUwB/h64Lcff17XAjcC+cT0fAR5LoZ4eYCrRJ7k18f97miz36vj181xgP+AntLjTe4o1vYLolE0vTfM1nfXvPl5W41cGz0fSmjSGaQwLUE+i8SvVF3+ABz8HuAXYDjwMnFl324lE0/G1nx8CdhNNcdYuX0qrhibbN6Lp8M3xZTXxabeyeE6yej6S1tNwn5NIoTOq3XqAdxPtP9EPfBc4KMff1/7A14EngKeB24EXplDPRUSfuOsvFxHt0zIALKpb9oPAJqL9S64F+lL6nbVUE/BTYLjhNf2DNGpK83c/xu9f41cO41c7NTXc5yQ0hmkMa6MeEo5fOneniIiISAEVubtTREREpGsppImIiIgUkEKaiIiISAEppImIiIgUkEKaiIiISAEppImIiIgUkEKaiIiISAEppImIiIgU0P8Hut8DXs9ag3AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1s = np.linspace(-0.2, 1.2, 100)\n",
    "x2s = np.linspace(-0.2, 1.2, 100)\n",
    "x1, x2 = np.meshgrid(x1s, x2s)\n",
    "\n",
    "z1 = mlp_xor(x1, x2, activation=heaviside)\n",
    "z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
    "\n",
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.contourf(x1, x2, z1)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: heaviside\", fontsize=14)\n",
    "plt.grid(True)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.contourf(x1, x2, z2)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: sigmoid\", fontsize=14)\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FNN for MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the Estimator API (formerly `tf.contrib.learn`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: `tf.examples.tutorials.mnist` is deprecated. We will use `tf.keras.datasets.mnist` instead. Moreover, the `tf.contrib.learn` API was promoted to `tf.estimators` and `tf.feature_columns`, and it has changed considerably. In particular, there is no `infer_real_valued_columns_from_input()` function or `SKCompat` class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "y_train = y_train.astype(np.int32)\n",
    "y_test = y_test.astype(np.int32)\n",
    "X_valid, X_train = X_train[:5000], X_train[5000:]\n",
    "y_valid, y_train = y_train[:5000], y_train[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpuflzeb_h\n",
      "INFO:tensorflow:Using config: {'_evaluation_master': '', '_session_config': None, '_model_dir': '/tmp/tmpuflzeb_h', '_task_type': 'worker', '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f4fcb4e15c0>, '_save_summary_steps': 100, '_is_chief': True, '_save_checkpoints_steps': None, '_log_step_count_steps': 100, '_master': '', '_service': None, '_keep_checkpoint_every_n_hours': 10000, '_task_id': 0, '_tf_random_seed': None, '_num_ps_replicas': 0, '_global_id_in_cluster': 0, '_train_distribute': None, '_num_worker_replicas': 1, '_save_checkpoints_secs': 600, '_keep_checkpoint_max': 5}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpuflzeb_h/model.ckpt.\n",
      "INFO:tensorflow:loss = 122.883514, step = 0\n",
      "INFO:tensorflow:global_step/sec: 480.267\n",
      "INFO:tensorflow:loss = 9.599711, step = 100 (0.209 sec)\n",
      "INFO:tensorflow:global_step/sec: 599.191\n",
      "INFO:tensorflow:loss = 19.580772, step = 200 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 640.184\n",
      "INFO:tensorflow:loss = 2.1866307, step = 300 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.395\n",
      "INFO:tensorflow:loss = 11.493204, step = 400 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.653\n",
      "INFO:tensorflow:loss = 4.0078278, step = 500 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 722.021\n",
      "INFO:tensorflow:loss = 10.612131, step = 600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.446\n",
      "INFO:tensorflow:loss = 6.692636, step = 700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.49\n",
      "INFO:tensorflow:loss = 4.2058306, step = 800 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.548\n",
      "INFO:tensorflow:loss = 9.13055, step = 900 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.506\n",
      "INFO:tensorflow:loss = 4.1445055, step = 1000 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.713\n",
      "INFO:tensorflow:loss = 8.442559, step = 1100 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.721\n",
      "INFO:tensorflow:loss = 1.4401194, step = 1200 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 659.992\n",
      "INFO:tensorflow:loss = 13.526959, step = 1300 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.683\n",
      "INFO:tensorflow:loss = 5.039109, step = 1400 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.167\n",
      "INFO:tensorflow:loss = 1.8074234, step = 1500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 700.915\n",
      "INFO:tensorflow:loss = 6.4867635, step = 1600 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.041\n",
      "INFO:tensorflow:loss = 0.5804969, step = 1700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.306\n",
      "INFO:tensorflow:loss = 1.5091155, step = 1800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.096\n",
      "INFO:tensorflow:loss = 3.6764488, step = 1900 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 687.762\n",
      "INFO:tensorflow:loss = 1.4820085, step = 2000 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 667.013\n",
      "INFO:tensorflow:loss = 1.2964534, step = 2100 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 704.008\n",
      "INFO:tensorflow:loss = 1.0711427, step = 2200 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.444\n",
      "INFO:tensorflow:loss = 2.6691673, step = 2300 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.387\n",
      "INFO:tensorflow:loss = 0.9668397, step = 2400 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.867\n",
      "INFO:tensorflow:loss = 1.4468323, step = 2500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 725.923\n",
      "INFO:tensorflow:loss = 1.8073778, step = 2600 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.432\n",
      "INFO:tensorflow:loss = 3.8904514, step = 2700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.886\n",
      "INFO:tensorflow:loss = 2.3015192, step = 2800 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.338\n",
      "INFO:tensorflow:loss = 4.671579, step = 2900 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.305\n",
      "INFO:tensorflow:loss = 1.3551085, step = 3000 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 647.185\n",
      "INFO:tensorflow:loss = 1.9145899, step = 3100 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 673.601\n",
      "INFO:tensorflow:loss = 1.0741266, step = 3200 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.531\n",
      "INFO:tensorflow:loss = 1.0713774, step = 3300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.421\n",
      "INFO:tensorflow:loss = 1.3074391, step = 3400 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 633.857\n",
      "INFO:tensorflow:loss = 2.0073137, step = 3500 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.287\n",
      "INFO:tensorflow:loss = 13.952677, step = 3600 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 639.094\n",
      "INFO:tensorflow:loss = 1.6767453, step = 3700 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.891\n",
      "INFO:tensorflow:loss = 0.27798674, step = 3800 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.154\n",
      "INFO:tensorflow:loss = 3.5524733, step = 3900 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.217\n",
      "INFO:tensorflow:loss = 0.6761815, step = 4000 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 744.721\n",
      "INFO:tensorflow:loss = 0.79083383, step = 4100 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 685.419\n",
      "INFO:tensorflow:loss = 1.3305103, step = 4200 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 632.625\n",
      "INFO:tensorflow:loss = 0.14447726, step = 4300 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.755\n",
      "INFO:tensorflow:loss = 1.8386902, step = 4400 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.428\n",
      "INFO:tensorflow:loss = 0.94889283, step = 4500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.376\n",
      "INFO:tensorflow:loss = 0.28424773, step = 4600 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.964\n",
      "INFO:tensorflow:loss = 3.266353, step = 4700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 742.054\n",
      "INFO:tensorflow:loss = 3.171119, step = 4800 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.657\n",
      "INFO:tensorflow:loss = 1.12006, step = 4900 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.952\n",
      "INFO:tensorflow:loss = 0.5669488, step = 5000 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.897\n",
      "INFO:tensorflow:loss = 0.3067366, step = 5100 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 631.438\n",
      "INFO:tensorflow:loss = 0.5437011, step = 5200 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 654.717\n",
      "INFO:tensorflow:loss = 0.25085437, step = 5300 (0.153 sec)\n",
      "INFO:tensorflow:global_step/sec: 658.829\n",
      "INFO:tensorflow:loss = 0.30891788, step = 5400 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.908\n",
      "INFO:tensorflow:loss = 0.8258436, step = 5500 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 574.211\n",
      "INFO:tensorflow:loss = 0.19280735, step = 5600 (0.173 sec)\n",
      "INFO:tensorflow:global_step/sec: 653.783\n",
      "INFO:tensorflow:loss = 0.17635345, step = 5700 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 635.343\n",
      "INFO:tensorflow:loss = 2.1531484, step = 5800 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 628.505\n",
      "INFO:tensorflow:loss = 1.3385924, step = 5900 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 743.577\n",
      "INFO:tensorflow:loss = 1.7493693, step = 6000 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 759.487\n",
      "INFO:tensorflow:loss = 1.3990214, step = 6100 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.001\n",
      "INFO:tensorflow:loss = 0.08636901, step = 6200 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 682.812\n",
      "INFO:tensorflow:loss = 1.2878852, step = 6300 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 668.255\n",
      "INFO:tensorflow:loss = 2.8647041, step = 6400 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 702.609\n",
      "INFO:tensorflow:loss = 0.38349468, step = 6500 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.91\n",
      "INFO:tensorflow:loss = 0.71950877, step = 6600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.603\n",
      "INFO:tensorflow:loss = 0.8812942, step = 6700 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 697.269\n",
      "INFO:tensorflow:loss = 0.5575855, step = 6800 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.281\n",
      "INFO:tensorflow:loss = 0.97547567, step = 6900 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.343\n",
      "INFO:tensorflow:loss = 0.10276175, step = 7000 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.462\n",
      "INFO:tensorflow:loss = 1.1644993, step = 7100 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 674.078\n",
      "INFO:tensorflow:loss = 0.87335706, step = 7200 (0.149 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 754.457\n",
      "INFO:tensorflow:loss = 0.02294194, step = 7300 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 665.539\n",
      "INFO:tensorflow:loss = 0.03988394, step = 7400 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.489\n",
      "INFO:tensorflow:loss = 0.35611078, step = 7500 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 642.162\n",
      "INFO:tensorflow:loss = 0.37163323, step = 7600 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.797\n",
      "INFO:tensorflow:loss = 0.2848217, step = 7700 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.344\n",
      "INFO:tensorflow:loss = 0.7845428, step = 7800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 765.77\n",
      "INFO:tensorflow:loss = 0.10574758, step = 7900 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.478\n",
      "INFO:tensorflow:loss = 0.69526076, step = 8000 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.877\n",
      "INFO:tensorflow:loss = 0.58130324, step = 8100 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.965\n",
      "INFO:tensorflow:loss = 0.31362545, step = 8200 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.223\n",
      "INFO:tensorflow:loss = 0.8284373, step = 8300 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 660.215\n",
      "INFO:tensorflow:loss = 0.82293755, step = 8400 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 650.197\n",
      "INFO:tensorflow:loss = 0.08349186, step = 8500 (0.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 686.024\n",
      "INFO:tensorflow:loss = 0.48411164, step = 8600 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.917\n",
      "INFO:tensorflow:loss = 0.35096797, step = 8700 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.152\n",
      "INFO:tensorflow:loss = 0.22725578, step = 8800 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.659\n",
      "INFO:tensorflow:loss = 0.0485073, step = 8900 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.385\n",
      "INFO:tensorflow:loss = 0.07985984, step = 9000 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.822\n",
      "INFO:tensorflow:loss = 0.0542172, step = 9100 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.122\n",
      "INFO:tensorflow:loss = 0.50927365, step = 9200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 681.55\n",
      "INFO:tensorflow:loss = 0.9489171, step = 9300 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 680.382\n",
      "INFO:tensorflow:loss = 0.060666546, step = 9400 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.193\n",
      "INFO:tensorflow:loss = 0.28517923, step = 9500 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 702.206\n",
      "INFO:tensorflow:loss = 0.20915146, step = 9600 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 678.51\n",
      "INFO:tensorflow:loss = 0.579557, step = 9700 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 593.939\n",
      "INFO:tensorflow:loss = 0.93460476, step = 9800 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 665.897\n",
      "INFO:tensorflow:loss = 0.047924682, step = 9900 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.483\n",
      "INFO:tensorflow:loss = 0.4922812, step = 10000 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 712.135\n",
      "INFO:tensorflow:loss = 0.047895633, step = 10100 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 645.55\n",
      "INFO:tensorflow:loss = 0.5398847, step = 10200 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 638.582\n",
      "INFO:tensorflow:loss = 0.15529627, step = 10300 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.801\n",
      "INFO:tensorflow:loss = 0.9468999, step = 10400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 699.759\n",
      "INFO:tensorflow:loss = 0.117633805, step = 10500 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 681.536\n",
      "INFO:tensorflow:loss = 2.7077622, step = 10600 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 639.102\n",
      "INFO:tensorflow:loss = 0.4701725, step = 10700 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 657.787\n",
      "INFO:tensorflow:loss = 0.036744572, step = 10800 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 610.198\n",
      "INFO:tensorflow:loss = 0.5130552, step = 10900 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 641.731\n",
      "INFO:tensorflow:loss = 0.10233489, step = 11000 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 676.149\n",
      "INFO:tensorflow:loss = 0.27252752, step = 11100 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 685.492\n",
      "INFO:tensorflow:loss = 0.03905679, step = 11200 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.538\n",
      "INFO:tensorflow:loss = 0.4037779, step = 11300 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.957\n",
      "INFO:tensorflow:loss = 0.07931179, step = 11400 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 690.435\n",
      "INFO:tensorflow:loss = 0.25471577, step = 11500 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.688\n",
      "INFO:tensorflow:loss = 0.095315784, step = 11600 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.292\n",
      "INFO:tensorflow:loss = 0.26888248, step = 11700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.634\n",
      "INFO:tensorflow:loss = 0.0791325, step = 11800 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.395\n",
      "INFO:tensorflow:loss = 0.22025564, step = 11900 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 674.538\n",
      "INFO:tensorflow:loss = 0.23279706, step = 12000 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.35\n",
      "INFO:tensorflow:loss = 0.12199997, step = 12100 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 666.45\n",
      "INFO:tensorflow:loss = 0.28451073, step = 12200 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.277\n",
      "INFO:tensorflow:loss = 0.118836306, step = 12300 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 697.121\n",
      "INFO:tensorflow:loss = 0.040629867, step = 12400 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 695.514\n",
      "INFO:tensorflow:loss = 0.12426446, step = 12500 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.65\n",
      "INFO:tensorflow:loss = 0.123113535, step = 12600 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.816\n",
      "INFO:tensorflow:loss = 0.08067431, step = 12700 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 677.035\n",
      "INFO:tensorflow:loss = 0.12166683, step = 12800 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.319\n",
      "INFO:tensorflow:loss = 0.44292378, step = 12900 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 663.768\n",
      "INFO:tensorflow:loss = 0.115894824, step = 13000 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.272\n",
      "INFO:tensorflow:loss = 0.48085117, step = 13100 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.079\n",
      "INFO:tensorflow:loss = 0.04950906, step = 13200 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.079\n",
      "INFO:tensorflow:loss = 0.22041032, step = 13300 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 743.21\n",
      "INFO:tensorflow:loss = 0.96188104, step = 13400 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 736.865\n",
      "INFO:tensorflow:loss = 0.48876682, step = 13500 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 753.917\n",
      "INFO:tensorflow:loss = 0.016977577, step = 13600 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.362\n",
      "INFO:tensorflow:loss = 0.21165565, step = 13700 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.43\n",
      "INFO:tensorflow:loss = 0.07759167, step = 13800 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.41\n",
      "INFO:tensorflow:loss = 0.30665547, step = 13900 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 628.631\n",
      "INFO:tensorflow:loss = 0.061501402, step = 14000 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 652.142\n",
      "INFO:tensorflow:loss = 0.22736073, step = 14100 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.068\n",
      "INFO:tensorflow:loss = 0.07389079, step = 14200 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 688.411\n",
      "INFO:tensorflow:loss = 0.29698288, step = 14300 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.249\n",
      "INFO:tensorflow:loss = 0.14422297, step = 14400 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.837\n",
      "INFO:tensorflow:loss = 0.014385125, step = 14500 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.481\n",
      "INFO:tensorflow:loss = 0.072385326, step = 14600 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 633.549\n",
      "INFO:tensorflow:loss = 0.2490507, step = 14700 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 655.833\n",
      "INFO:tensorflow:loss = 0.13730074, step = 14800 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.018\n",
      "INFO:tensorflow:loss = 0.094497725, step = 14900 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 648.228\n",
      "INFO:tensorflow:loss = 0.02053002, step = 15000 (0.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.368\n",
      "INFO:tensorflow:loss = 0.07519817, step = 15100 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.654\n",
      "INFO:tensorflow:loss = 5.0296702, step = 15200 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.54\n",
      "INFO:tensorflow:loss = 0.0641305, step = 15300 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 706.583\n",
      "INFO:tensorflow:loss = 0.046975322, step = 15400 (0.142 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 760.143\n",
      "INFO:tensorflow:loss = 0.19096217, step = 15500 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 729.355\n",
      "INFO:tensorflow:loss = 0.07447162, step = 15600 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.879\n",
      "INFO:tensorflow:loss = 0.008595724, step = 15700 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 725.15\n",
      "INFO:tensorflow:loss = 0.02350374, step = 15800 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.054\n",
      "INFO:tensorflow:loss = 0.23393407, step = 15900 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 686.044\n",
      "INFO:tensorflow:loss = 0.052663296, step = 16000 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 571.582\n",
      "INFO:tensorflow:loss = 0.07737052, step = 16100 (0.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.862\n",
      "INFO:tensorflow:loss = 0.059392054, step = 16200 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 680.005\n",
      "INFO:tensorflow:loss = 0.112042084, step = 16300 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.16\n",
      "INFO:tensorflow:loss = 0.019262556, step = 16400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 747.072\n",
      "INFO:tensorflow:loss = 0.11752561, step = 16500 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.132\n",
      "INFO:tensorflow:loss = 0.23796175, step = 16600 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.207\n",
      "INFO:tensorflow:loss = 0.2813179, step = 16700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.658\n",
      "INFO:tensorflow:loss = 0.08527176, step = 16800 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.729\n",
      "INFO:tensorflow:loss = 0.0708789, step = 16900 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 780.434\n",
      "INFO:tensorflow:loss = 0.10494808, step = 17000 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 802.2\n",
      "INFO:tensorflow:loss = 0.053045884, step = 17100 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.93\n",
      "INFO:tensorflow:loss = 0.019496916, step = 17200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 777.573\n",
      "INFO:tensorflow:loss = 0.2686581, step = 17300 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 798.333\n",
      "INFO:tensorflow:loss = 0.0589393, step = 17400 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.037\n",
      "INFO:tensorflow:loss = 0.04246951, step = 17500 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 781.962\n",
      "INFO:tensorflow:loss = 0.1414603, step = 17600 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 637.081\n",
      "INFO:tensorflow:loss = 0.03246697, step = 17700 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.478\n",
      "INFO:tensorflow:loss = 0.1726821, step = 17800 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 670.541\n",
      "INFO:tensorflow:loss = 0.16460724, step = 17900 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.135\n",
      "INFO:tensorflow:loss = 0.2628479, step = 18000 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.683\n",
      "INFO:tensorflow:loss = 0.092953764, step = 18100 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.433\n",
      "INFO:tensorflow:loss = 0.04855925, step = 18200 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 724.314\n",
      "INFO:tensorflow:loss = 0.1251901, step = 18300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.447\n",
      "INFO:tensorflow:loss = 0.07948378, step = 18400 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.474\n",
      "INFO:tensorflow:loss = 0.0709505, step = 18500 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 681.306\n",
      "INFO:tensorflow:loss = 0.047453385, step = 18600 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.621\n",
      "INFO:tensorflow:loss = 0.25552556, step = 18700 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 782.048\n",
      "INFO:tensorflow:loss = 0.09249738, step = 18800 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 592.506\n",
      "INFO:tensorflow:loss = 0.013389738, step = 18900 (0.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 673.123\n",
      "INFO:tensorflow:loss = 0.13284124, step = 19000 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.839\n",
      "INFO:tensorflow:loss = 0.1085002, step = 19100 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.734\n",
      "INFO:tensorflow:loss = 0.2795027, step = 19200 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 630.326\n",
      "INFO:tensorflow:loss = 0.04235497, step = 19300 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 744.772\n",
      "INFO:tensorflow:loss = 0.184058, step = 19400 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.66\n",
      "INFO:tensorflow:loss = 0.10944146, step = 19500 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 682.055\n",
      "INFO:tensorflow:loss = 0.61868834, step = 19600 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.81\n",
      "INFO:tensorflow:loss = 0.013134284, step = 19700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.217\n",
      "INFO:tensorflow:loss = 0.07665045, step = 19800 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 765.268\n",
      "INFO:tensorflow:loss = 0.13330126, step = 19900 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.119\n",
      "INFO:tensorflow:loss = 0.1017472, step = 20000 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.664\n",
      "INFO:tensorflow:loss = 0.32613552, step = 20100 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.936\n",
      "INFO:tensorflow:loss = 0.08524483, step = 20200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.554\n",
      "INFO:tensorflow:loss = 0.10158778, step = 20300 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 776.959\n",
      "INFO:tensorflow:loss = 0.029059863, step = 20400 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 788.694\n",
      "INFO:tensorflow:loss = 0.012308217, step = 20500 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 782.133\n",
      "INFO:tensorflow:loss = 0.059527285, step = 20600 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.991\n",
      "INFO:tensorflow:loss = 0.008613001, step = 20700 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 600.703\n",
      "INFO:tensorflow:loss = 0.006468304, step = 20800 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 555.84\n",
      "INFO:tensorflow:loss = 0.041916292, step = 20900 (0.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 636.831\n",
      "INFO:tensorflow:loss = 0.004384146, step = 21000 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 683.558\n",
      "INFO:tensorflow:loss = 0.024349477, step = 21100 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 640.381\n",
      "INFO:tensorflow:loss = 0.04808281, step = 21200 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.647\n",
      "INFO:tensorflow:loss = 0.12497167, step = 21300 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.897\n",
      "INFO:tensorflow:loss = 0.037052717, step = 21400 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.997\n",
      "INFO:tensorflow:loss = 0.006926307, step = 21500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.32\n",
      "INFO:tensorflow:loss = 0.074007586, step = 21600 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 718.055\n",
      "INFO:tensorflow:loss = 0.10291884, step = 21700 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.981\n",
      "INFO:tensorflow:loss = 0.012438581, step = 21800 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 765.381\n",
      "INFO:tensorflow:loss = 0.0038113636, step = 21900 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.057\n",
      "INFO:tensorflow:loss = 0.03924905, step = 22000 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.258\n",
      "INFO:tensorflow:loss = 0.0941266, step = 22100 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.376\n",
      "INFO:tensorflow:loss = 0.0047066705, step = 22200 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.068\n",
      "INFO:tensorflow:loss = 0.049610026, step = 22300 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.656\n",
      "INFO:tensorflow:loss = 0.026602667, step = 22400 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.336\n",
      "INFO:tensorflow:loss = 0.037756026, step = 22500 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.671\n",
      "INFO:tensorflow:loss = 0.009034704, step = 22600 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 696.592\n",
      "INFO:tensorflow:loss = 0.121186815, step = 22700 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.805\n",
      "INFO:tensorflow:loss = 0.018444436, step = 22800 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.769\n",
      "INFO:tensorflow:loss = 0.15817365, step = 22900 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.057\n",
      "INFO:tensorflow:loss = 0.07102643, step = 23000 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.982\n",
      "INFO:tensorflow:loss = 0.053571053, step = 23100 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.232\n",
      "INFO:tensorflow:loss = 0.0041952957, step = 23200 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.255\n",
      "INFO:tensorflow:loss = 0.07324758, step = 23300 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 698.442\n",
      "INFO:tensorflow:loss = 0.013589903, step = 23400 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.646\n",
      "INFO:tensorflow:loss = 0.10314279, step = 23500 (0.149 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 720.016\n",
      "INFO:tensorflow:loss = 0.05611958, step = 23600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 670.816\n",
      "INFO:tensorflow:loss = 0.040629435, step = 23700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.358\n",
      "INFO:tensorflow:loss = 0.039468687, step = 23800 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 591.633\n",
      "INFO:tensorflow:loss = 0.07725768, step = 23900 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 626.386\n",
      "INFO:tensorflow:loss = 0.07133426, step = 24000 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.443\n",
      "INFO:tensorflow:loss = 0.01620267, step = 24100 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 757.569\n",
      "INFO:tensorflow:loss = 0.033674255, step = 24200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 759.175\n",
      "INFO:tensorflow:loss = 0.02904886, step = 24300 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 643.196\n",
      "INFO:tensorflow:loss = 0.021125063, step = 24400 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.936\n",
      "INFO:tensorflow:loss = 0.010064006, step = 24500 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.595\n",
      "INFO:tensorflow:loss = 0.22093584, step = 24600 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 642.635\n",
      "INFO:tensorflow:loss = 0.04337728, step = 24700 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 642.833\n",
      "INFO:tensorflow:loss = 0.017090034, step = 24800 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 648.077\n",
      "INFO:tensorflow:loss = 0.058798455, step = 24900 (0.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 611.414\n",
      "INFO:tensorflow:loss = 0.026018398, step = 25000 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 637.37\n",
      "INFO:tensorflow:loss = 0.05266387, step = 25100 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 662.643\n",
      "INFO:tensorflow:loss = 0.02904391, step = 25200 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.739\n",
      "INFO:tensorflow:loss = 0.03309081, step = 25300 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 598.224\n",
      "INFO:tensorflow:loss = 0.114326894, step = 25400 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 663.694\n",
      "INFO:tensorflow:loss = 0.038087483, step = 25500 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 636.034\n",
      "INFO:tensorflow:loss = 0.042078175, step = 25600 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 587.527\n",
      "INFO:tensorflow:loss = 0.021071143, step = 25700 (0.170 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.395\n",
      "INFO:tensorflow:loss = 0.039268643, step = 25800 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.829\n",
      "INFO:tensorflow:loss = 0.00580811, step = 25900 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 636.116\n",
      "INFO:tensorflow:loss = 0.06418324, step = 26000 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.763\n",
      "INFO:tensorflow:loss = 0.030529939, step = 26100 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 592.966\n",
      "INFO:tensorflow:loss = 0.019166447, step = 26200 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 589.27\n",
      "INFO:tensorflow:loss = 0.0116247, step = 26300 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 695.168\n",
      "INFO:tensorflow:loss = 0.030719474, step = 26400 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.142\n",
      "INFO:tensorflow:loss = 0.11685369, step = 26500 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.473\n",
      "INFO:tensorflow:loss = 0.01657794, step = 26600 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 671.327\n",
      "INFO:tensorflow:loss = 0.02757436, step = 26700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 668.14\n",
      "INFO:tensorflow:loss = 0.013070018, step = 26800 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 771.784\n",
      "INFO:tensorflow:loss = 0.16727, step = 26900 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.783\n",
      "INFO:tensorflow:loss = 0.026858501, step = 27000 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 806.951\n",
      "INFO:tensorflow:loss = 0.020017473, step = 27100 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 792.151\n",
      "INFO:tensorflow:loss = 0.0063375104, step = 27200 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 743.85\n",
      "INFO:tensorflow:loss = 0.038745653, step = 27300 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.696\n",
      "INFO:tensorflow:loss = 0.07117924, step = 27400 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 718.526\n",
      "INFO:tensorflow:loss = 0.23104118, step = 27500 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.104\n",
      "INFO:tensorflow:loss = 0.045554783, step = 27600 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.787\n",
      "INFO:tensorflow:loss = 0.054858394, step = 27700 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.613\n",
      "INFO:tensorflow:loss = 0.1919387, step = 27800 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 676.335\n",
      "INFO:tensorflow:loss = 0.035167128, step = 27900 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.045\n",
      "INFO:tensorflow:loss = 0.06809587, step = 28000 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.169\n",
      "INFO:tensorflow:loss = 0.020983905, step = 28100 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.403\n",
      "INFO:tensorflow:loss = 0.009154754, step = 28200 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.517\n",
      "INFO:tensorflow:loss = 0.013444103, step = 28300 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.488\n",
      "INFO:tensorflow:loss = 0.019219223, step = 28400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.509\n",
      "INFO:tensorflow:loss = 0.0007980311, step = 28500 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.906\n",
      "INFO:tensorflow:loss = 0.037370995, step = 28600 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 693.067\n",
      "INFO:tensorflow:loss = 0.034441676, step = 28700 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.073\n",
      "INFO:tensorflow:loss = 0.019205665, step = 28800 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.271\n",
      "INFO:tensorflow:loss = 0.04330594, step = 28900 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.943\n",
      "INFO:tensorflow:loss = 0.02258328, step = 29000 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.761\n",
      "INFO:tensorflow:loss = 0.033814937, step = 29100 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 736.08\n",
      "INFO:tensorflow:loss = 0.06819173, step = 29200 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.266\n",
      "INFO:tensorflow:loss = 0.03306775, step = 29300 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.808\n",
      "INFO:tensorflow:loss = 0.044940256, step = 29400 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.514\n",
      "INFO:tensorflow:loss = 0.041515682, step = 29500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.502\n",
      "INFO:tensorflow:loss = 0.0137351295, step = 29600 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 726.773\n",
      "INFO:tensorflow:loss = 0.058922045, step = 29700 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 772.537\n",
      "INFO:tensorflow:loss = 0.004406413, step = 29800 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.1\n",
      "INFO:tensorflow:loss = 0.02618828, step = 29900 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.709\n",
      "INFO:tensorflow:loss = 0.035493106, step = 30000 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.042\n",
      "INFO:tensorflow:loss = 0.032594115, step = 30100 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 774.994\n",
      "INFO:tensorflow:loss = 0.012513466, step = 30200 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 727.626\n",
      "INFO:tensorflow:loss = 0.030434133, step = 30300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.067\n",
      "INFO:tensorflow:loss = 0.009930114, step = 30400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 740.545\n",
      "INFO:tensorflow:loss = 0.012347832, step = 30500 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.107\n",
      "INFO:tensorflow:loss = 0.008713552, step = 30600 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.775\n",
      "INFO:tensorflow:loss = 0.01328897, step = 30700 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 700.529\n",
      "INFO:tensorflow:loss = 0.005648759, step = 30800 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.498\n",
      "INFO:tensorflow:loss = 0.045800988, step = 30900 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 759.22\n",
      "INFO:tensorflow:loss = 0.0049998416, step = 31000 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.944\n",
      "INFO:tensorflow:loss = 0.025347317, step = 31100 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.877\n",
      "INFO:tensorflow:loss = 0.02644661, step = 31200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 722.356\n",
      "INFO:tensorflow:loss = 0.012870766, step = 31300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 659.329\n",
      "INFO:tensorflow:loss = 0.030599056, step = 31400 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 724.764\n",
      "INFO:tensorflow:loss = 0.09482564, step = 31500 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 738.252\n",
      "INFO:tensorflow:loss = 0.019195255, step = 31600 (0.135 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 724.457\n",
      "INFO:tensorflow:loss = 0.011149004, step = 31700 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.343\n",
      "INFO:tensorflow:loss = 0.011274818, step = 31800 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.722\n",
      "INFO:tensorflow:loss = 0.016880032, step = 31900 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 686.074\n",
      "INFO:tensorflow:loss = 0.034965005, step = 32000 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 747.228\n",
      "INFO:tensorflow:loss = 0.0711512, step = 32100 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.061\n",
      "INFO:tensorflow:loss = 0.020935431, step = 32200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.156\n",
      "INFO:tensorflow:loss = 0.05625681, step = 32300 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 752.545\n",
      "INFO:tensorflow:loss = 0.008945845, step = 32400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 774.541\n",
      "INFO:tensorflow:loss = 0.0025368293, step = 32500 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.982\n",
      "INFO:tensorflow:loss = 0.025532966, step = 32600 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.828\n",
      "INFO:tensorflow:loss = 0.024839351, step = 32700 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 740.441\n",
      "INFO:tensorflow:loss = 0.05547411, step = 32800 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.51\n",
      "INFO:tensorflow:loss = 0.09110242, step = 32900 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.617\n",
      "INFO:tensorflow:loss = 0.027063178, step = 33000 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 704.674\n",
      "INFO:tensorflow:loss = 0.14633262, step = 33100 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.149\n",
      "INFO:tensorflow:loss = 0.007330508, step = 33200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.544\n",
      "INFO:tensorflow:loss = 0.008549547, step = 33300 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.367\n",
      "INFO:tensorflow:loss = 0.024134047, step = 33400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.464\n",
      "INFO:tensorflow:loss = 0.0074961022, step = 33500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.015\n",
      "INFO:tensorflow:loss = 0.018352449, step = 33600 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 790.108\n",
      "INFO:tensorflow:loss = 0.016640741, step = 33700 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 781.526\n",
      "INFO:tensorflow:loss = 0.0017197947, step = 33800 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 785.147\n",
      "INFO:tensorflow:loss = 0.0111332275, step = 33900 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.909\n",
      "INFO:tensorflow:loss = 0.040406324, step = 34000 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 772.981\n",
      "INFO:tensorflow:loss = 0.012423429, step = 34100 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 781.531\n",
      "INFO:tensorflow:loss = 0.015810683, step = 34200 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.78\n",
      "INFO:tensorflow:loss = 0.0105865635, step = 34300 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.963\n",
      "INFO:tensorflow:loss = 0.022520129, step = 34400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 687.737\n",
      "INFO:tensorflow:loss = 0.007853265, step = 34500 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.826\n",
      "INFO:tensorflow:loss = 0.017153783, step = 34600 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.16\n",
      "INFO:tensorflow:loss = 0.04800478, step = 34700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 687.501\n",
      "INFO:tensorflow:loss = 0.022419866, step = 34800 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.493\n",
      "INFO:tensorflow:loss = 0.0017927951, step = 34900 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.438\n",
      "INFO:tensorflow:loss = 0.04672495, step = 35000 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 757.189\n",
      "INFO:tensorflow:loss = 0.0890024, step = 35100 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.348\n",
      "INFO:tensorflow:loss = 0.0034388623, step = 35200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.144\n",
      "INFO:tensorflow:loss = 0.033513878, step = 35300 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.169\n",
      "INFO:tensorflow:loss = 0.0026553052, step = 35400 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.332\n",
      "INFO:tensorflow:loss = 0.040838942, step = 35500 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 693.586\n",
      "INFO:tensorflow:loss = 0.023297546, step = 35600 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.726\n",
      "INFO:tensorflow:loss = 0.039266217, step = 35700 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 800.56\n",
      "INFO:tensorflow:loss = 0.006094549, step = 35800 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.842\n",
      "INFO:tensorflow:loss = 0.024597563, step = 35900 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 791.645\n",
      "INFO:tensorflow:loss = 0.0005974211, step = 36000 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.571\n",
      "INFO:tensorflow:loss = 0.008686724, step = 36100 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.966\n",
      "INFO:tensorflow:loss = 0.003942257, step = 36200 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 786.328\n",
      "INFO:tensorflow:loss = 0.012746293, step = 36300 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.898\n",
      "INFO:tensorflow:loss = 0.03244071, step = 36400 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.438\n",
      "INFO:tensorflow:loss = 0.0060920594, step = 36500 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 789.399\n",
      "INFO:tensorflow:loss = 0.02364352, step = 36600 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 798.938\n",
      "INFO:tensorflow:loss = 0.03801505, step = 36700 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 643.811\n",
      "INFO:tensorflow:loss = 0.016948957, step = 36800 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.83\n",
      "INFO:tensorflow:loss = 0.017786317, step = 36900 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 543.587\n",
      "INFO:tensorflow:loss = 0.058662392, step = 37000 (0.183 sec)\n",
      "INFO:tensorflow:global_step/sec: 632.46\n",
      "INFO:tensorflow:loss = 0.014718251, step = 37100 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.433\n",
      "INFO:tensorflow:loss = 0.047604885, step = 37200 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 786.68\n",
      "INFO:tensorflow:loss = 0.002843613, step = 37300 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 799.111\n",
      "INFO:tensorflow:loss = 0.023884777, step = 37400 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.131\n",
      "INFO:tensorflow:loss = 0.017163115, step = 37500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 633.787\n",
      "INFO:tensorflow:loss = 0.026272407, step = 37600 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.891\n",
      "INFO:tensorflow:loss = 0.0028121169, step = 37700 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.086\n",
      "INFO:tensorflow:loss = 0.016791945, step = 37800 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.247\n",
      "INFO:tensorflow:loss = 0.047742378, step = 37900 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 682.181\n",
      "INFO:tensorflow:loss = 0.04361719, step = 38000 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 786.376\n",
      "INFO:tensorflow:loss = 0.038325347, step = 38100 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 804.283\n",
      "INFO:tensorflow:loss = 0.010711223, step = 38200 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 770.032\n",
      "INFO:tensorflow:loss = 0.11263705, step = 38300 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 759.344\n",
      "INFO:tensorflow:loss = 0.06331622, step = 38400 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 793.153\n",
      "INFO:tensorflow:loss = 0.03488349, step = 38500 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.129\n",
      "INFO:tensorflow:loss = 0.024869196, step = 38600 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.015\n",
      "INFO:tensorflow:loss = 0.049668543, step = 38700 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 795.537\n",
      "INFO:tensorflow:loss = 0.024653997, step = 38800 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.715\n",
      "INFO:tensorflow:loss = 0.0064418544, step = 38900 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 805.211\n",
      "INFO:tensorflow:loss = 0.025671637, step = 39000 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.917\n",
      "INFO:tensorflow:loss = 0.007240239, step = 39100 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.067\n",
      "INFO:tensorflow:loss = 0.0044421586, step = 39200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.837\n",
      "INFO:tensorflow:loss = 0.024619173, step = 39300 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.662\n",
      "INFO:tensorflow:loss = 0.012689214, step = 39400 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.614\n",
      "INFO:tensorflow:loss = 0.018057879, step = 39500 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.068\n",
      "INFO:tensorflow:loss = 0.02785282, step = 39600 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 654.979\n",
      "INFO:tensorflow:loss = 0.012543851, step = 39700 (0.152 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 731.693\n",
      "INFO:tensorflow:loss = 0.034165256, step = 39800 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 712.66\n",
      "INFO:tensorflow:loss = 0.018817844, step = 39900 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 767.888\n",
      "INFO:tensorflow:loss = 0.064586915, step = 40000 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 682.678\n",
      "INFO:tensorflow:loss = 0.039944634, step = 40100 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 637.928\n",
      "INFO:tensorflow:loss = 0.006592812, step = 40200 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 654.42\n",
      "INFO:tensorflow:loss = 0.0012000705, step = 40300 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.869\n",
      "INFO:tensorflow:loss = 0.0021813547, step = 40400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.684\n",
      "INFO:tensorflow:loss = 0.020016534, step = 40500 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 722.897\n",
      "INFO:tensorflow:loss = 0.010720066, step = 40600 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.822\n",
      "INFO:tensorflow:loss = 0.05658368, step = 40700 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.754\n",
      "INFO:tensorflow:loss = 0.013007379, step = 40800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.558\n",
      "INFO:tensorflow:loss = 0.0246561, step = 40900 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 658.625\n",
      "INFO:tensorflow:loss = 0.01263381, step = 41000 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.157\n",
      "INFO:tensorflow:loss = 0.004255068, step = 41100 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.883\n",
      "INFO:tensorflow:loss = 0.015441139, step = 41200 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 797.458\n",
      "INFO:tensorflow:loss = 0.010338774, step = 41300 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 747.132\n",
      "INFO:tensorflow:loss = 0.021103038, step = 41400 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.845\n",
      "INFO:tensorflow:loss = 0.025617313, step = 41500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.583\n",
      "INFO:tensorflow:loss = 0.013572226, step = 41600 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.386\n",
      "INFO:tensorflow:loss = 0.0033336552, step = 41700 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 782.906\n",
      "INFO:tensorflow:loss = 0.053168457, step = 41800 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.6\n",
      "INFO:tensorflow:loss = 0.029688315, step = 41900 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 788.136\n",
      "INFO:tensorflow:loss = 0.01654516, step = 42000 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 738.526\n",
      "INFO:tensorflow:loss = 0.015659308, step = 42100 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 777.578\n",
      "INFO:tensorflow:loss = 0.01840079, step = 42200 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 786.416\n",
      "INFO:tensorflow:loss = 0.021663003, step = 42300 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.347\n",
      "INFO:tensorflow:loss = 0.011599571, step = 42400 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.321\n",
      "INFO:tensorflow:loss = 0.0044469903, step = 42500 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.549\n",
      "INFO:tensorflow:loss = 0.0019147585, step = 42600 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 771.429\n",
      "INFO:tensorflow:loss = 0.0054854164, step = 42700 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 793.871\n",
      "INFO:tensorflow:loss = 0.0017117725, step = 42800 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 770.1\n",
      "INFO:tensorflow:loss = 0.012048513, step = 42900 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 744.636\n",
      "INFO:tensorflow:loss = 0.06634566, step = 43000 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 696.882\n",
      "INFO:tensorflow:loss = 0.0003919307, step = 43100 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.516\n",
      "INFO:tensorflow:loss = 0.06582007, step = 43200 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 699.244\n",
      "INFO:tensorflow:loss = 0.0038124803, step = 43300 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 792.079\n",
      "INFO:tensorflow:loss = 0.003364585, step = 43400 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 753.586\n",
      "INFO:tensorflow:loss = 0.00725976, step = 43500 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.951\n",
      "INFO:tensorflow:loss = 0.024148291, step = 43600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 770.384\n",
      "INFO:tensorflow:loss = 0.013779048, step = 43700 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 799.363\n",
      "INFO:tensorflow:loss = 0.014951154, step = 43800 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 791.774\n",
      "INFO:tensorflow:loss = 0.0015594304, step = 43900 (0.126 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 44000 into /tmp/tmpuflzeb_h/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0012097486.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.estimator.canned.dnn.DNNClassifier at 0x7f4f62b23be0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_cols = [tf.feature_column.numeric_column(\"X\", shape=[28 * 28])]\n",
    "dnn_clf = tf.estimator.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                     feature_columns=feature_cols)\n",
    "\n",
    "input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_train}, y=y_train, num_epochs=40, batch_size=50, shuffle=True)\n",
    "dnn_clf.train(input_fn=input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-18-19:12:49\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpuflzeb_h/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-18-19:12:50\n",
      "INFO:tensorflow:Saving dict for global step 44000: accuracy = 0.9798, average_loss = 0.10096103, global_step = 44000, loss = 12.779877\n"
     ]
    }
   ],
   "source": [
    "test_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_test}, y=y_test, shuffle=False)\n",
    "eval_results = dnn_clf.evaluate(input_fn=test_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9798,\n",
       " 'average_loss': 0.10096103,\n",
       " 'global_step': 44000,\n",
       " 'loss': 12.779877}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpuflzeb_h/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'class_ids': array([7]),\n",
       " 'classes': array([b'7'], dtype=object),\n",
       " 'logits': array([ -3.809414 ,  -4.1564407,  -0.426081 ,   3.2636993, -11.065331 ,\n",
       "         -8.790985 , -10.436305 ,  19.935707 ,  -6.9282775,   2.2807484],\n",
       "       dtype=float32),\n",
       " 'probabilities': array([4.8710768e-11, 3.4428106e-11, 1.4354495e-09, 5.7469666e-08,\n",
       "        3.4389070e-14, 3.3431518e-13, 6.4506329e-14, 1.0000000e+00,\n",
       "        2.1533745e-12, 2.1505466e-08], dtype=float32)}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_iter = dnn_clf.predict(input_fn=test_input_fn)\n",
    "y_pred = list(y_pred_iter)\n",
    "y_pred[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Using plain TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neuron_layer(X, n_neurons, name, activation=None):\n",
    "    with tf.name_scope(name):\n",
    "        n_inputs = int(X.get_shape()[1])\n",
    "        stddev = 2 / np.sqrt(n_inputs)\n",
    "        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n",
    "        W = tf.Variable(init, name=\"kernel\")\n",
    "        b = tf.Variable(tf.zeros([n_neurons]), name=\"bias\")\n",
    "        Z = tf.matmul(X, W) + b\n",
    "        if activation is not None:\n",
    "            return activation(Z)\n",
    "        else:\n",
    "            return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = neuron_layer(X, n_hidden1, name=\"hidden1\",\n",
    "                           activation=tf.nn.relu)\n",
    "    hidden2 = neuron_layer(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                           activation=tf.nn.relu)\n",
    "    logits = neuron_layer(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_batch(X, y, batch_size):\n",
    "    rnd_idx = np.random.permutation(len(X))\n",
    "    n_batches = len(X) // batch_size\n",
    "    for batch_idx in np.array_split(rnd_idx, n_batches):\n",
    "        X_batch, y_batch = X[batch_idx], y[batch_idx]\n",
    "        yield X_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Batch accuracy: 0.9 Val accuracy: 0.9146\n",
      "1 Batch accuracy: 0.92 Val accuracy: 0.936\n",
      "2 Batch accuracy: 0.96 Val accuracy: 0.945\n",
      "3 Batch accuracy: 0.92 Val accuracy: 0.9512\n",
      "4 Batch accuracy: 0.98 Val accuracy: 0.9558\n",
      "5 Batch accuracy: 0.96 Val accuracy: 0.9566\n",
      "6 Batch accuracy: 1.0 Val accuracy: 0.9612\n",
      "7 Batch accuracy: 0.94 Val accuracy: 0.963\n",
      "8 Batch accuracy: 0.98 Val accuracy: 0.9652\n",
      "9 Batch accuracy: 0.96 Val accuracy: 0.966\n",
      "10 Batch accuracy: 0.92 Val accuracy: 0.9688\n",
      "11 Batch accuracy: 0.98 Val accuracy: 0.969\n",
      "12 Batch accuracy: 0.98 Val accuracy: 0.967\n",
      "13 Batch accuracy: 0.98 Val accuracy: 0.9706\n",
      "14 Batch accuracy: 1.0 Val accuracy: 0.9714\n",
      "15 Batch accuracy: 0.94 Val accuracy: 0.9732\n",
      "16 Batch accuracy: 1.0 Val accuracy: 0.9736\n",
      "17 Batch accuracy: 1.0 Val accuracy: 0.9742\n",
      "18 Batch accuracy: 1.0 Val accuracy: 0.9746\n",
      "19 Batch accuracy: 0.98 Val accuracy: 0.9748\n",
      "20 Batch accuracy: 1.0 Val accuracy: 0.9752\n",
      "21 Batch accuracy: 1.0 Val accuracy: 0.9752\n",
      "22 Batch accuracy: 0.98 Val accuracy: 0.9764\n",
      "23 Batch accuracy: 0.98 Val accuracy: 0.9752\n",
      "24 Batch accuracy: 0.98 Val accuracy: 0.9772\n",
      "25 Batch accuracy: 1.0 Val accuracy: 0.977\n",
      "26 Batch accuracy: 0.98 Val accuracy: 0.9778\n",
      "27 Batch accuracy: 1.0 Val accuracy: 0.9774\n",
      "28 Batch accuracy: 0.96 Val accuracy: 0.9754\n",
      "29 Batch accuracy: 0.98 Val accuracy: 0.9776\n",
      "30 Batch accuracy: 1.0 Val accuracy: 0.9756\n",
      "31 Batch accuracy: 0.98 Val accuracy: 0.9772\n",
      "32 Batch accuracy: 0.98 Val accuracy: 0.9772\n",
      "33 Batch accuracy: 0.98 Val accuracy: 0.979\n",
      "34 Batch accuracy: 1.0 Val accuracy: 0.9784\n",
      "35 Batch accuracy: 1.0 Val accuracy: 0.9778\n",
      "36 Batch accuracy: 0.98 Val accuracy: 0.978\n",
      "37 Batch accuracy: 1.0 Val accuracy: 0.9776\n",
      "38 Batch accuracy: 1.0 Val accuracy: 0.9792\n",
      "39 Batch accuracy: 1.0 Val accuracy: 0.9776\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"Batch accuracy:\", acc_batch, \"Val accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\") # or better, use save_path\n",
    "    X_new_scaled = X_test[:20]\n",
    "    Z = logits.eval(feed_dict={X: X_new_scaled})\n",
    "    y_pred = np.argmax(Z, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted classes: [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n",
      "Actual classes:    [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n"
     ]
    }
   ],
   "source": [
    "print(\"Predicted classes:\", y_pred)\n",
    "print(\"Actual classes:   \", y_test[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow_graph_in_jupyter import show_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.2851015593374667&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0714285746216774\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden1/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;dnn/hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.1154700517654419\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 21\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/hidden2/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;dnn/hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.20000000298023224\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 37\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;dnn/outputs/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;dnn/outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^dnn/hidden1/bias/Assign&quot;\\n  input: &quot;^dnn/hidden1/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden2/bias/Assign&quot;\\n  input: &quot;^dnn/hidden2/kernel/Assign&quot;\\n  input: &quot;^dnn/outputs/bias/Assign&quot;\\n  input: &quot;^dnn/outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.2851015593374667&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using `dense()` instead of `neuron_layer()`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: previous releases of the book used `tensorflow.contrib.layers.fully_connected()` rather than `tf.layers.dense()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dense()`, because anything in the contrib module may change or be deleted without notice. The `dense()` function is almost identical to the `fully_connected()` function, except for a few minor differences:\n",
    "* several parameters are renamed: `scope` becomes `name`, `activation_fn` becomes `activation` (and similarly the `_fn` suffix is removed from other parameters such as `normalizer_fn`), `weights_initializer` becomes `kernel_initializer`, etc.\n",
    "* the default `activation` is now `None` rather than `tf.nn.relu`.\n",
    "* a few more differences are presented in chapter 11."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "    y_proba = tf.nn.softmax(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Batch accuracy: 0.9 Validation accuracy: 0.9024\n",
      "1 Batch accuracy: 0.92 Validation accuracy: 0.9254\n",
      "2 Batch accuracy: 0.94 Validation accuracy: 0.9372\n",
      "3 Batch accuracy: 0.9 Validation accuracy: 0.9416\n",
      "4 Batch accuracy: 0.94 Validation accuracy: 0.9472\n",
      "5 Batch accuracy: 0.94 Validation accuracy: 0.9512\n",
      "6 Batch accuracy: 1.0 Validation accuracy: 0.9548\n",
      "7 Batch accuracy: 0.94 Validation accuracy: 0.961\n",
      "8 Batch accuracy: 0.96 Validation accuracy: 0.962\n",
      "9 Batch accuracy: 0.94 Validation accuracy: 0.9648\n",
      "10 Batch accuracy: 0.92 Validation accuracy: 0.9656\n",
      "11 Batch accuracy: 0.98 Validation accuracy: 0.9668\n",
      "12 Batch accuracy: 0.98 Validation accuracy: 0.9684\n",
      "13 Batch accuracy: 0.98 Validation accuracy: 0.9702\n",
      "14 Batch accuracy: 1.0 Validation accuracy: 0.9696\n",
      "15 Batch accuracy: 0.94 Validation accuracy: 0.9718\n",
      "16 Batch accuracy: 0.98 Validation accuracy: 0.9728\n",
      "17 Batch accuracy: 1.0 Validation accuracy: 0.973\n",
      "18 Batch accuracy: 0.98 Validation accuracy: 0.9748\n",
      "19 Batch accuracy: 0.98 Validation accuracy: 0.9756\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "n_batches = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"Batch accuracy:\", acc_batch, \"Validation accuracy:\", acc_valid)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.7224827313584268&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 22\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 39\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/Softmax&quot;\\n  op: &quot;Softmax&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.7224827313584268&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 8."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "See appendix A."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Train a deep MLP on the MNIST dataset and see if you can get over 98% precision. Just like in the last exercise of chapter 9, try adding all the bells and whistles (i.e., save checkpoints, restore the last checkpoint in case of an interruption, add summaries, plot learning curves using TensorBoard, and so on)._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's create the deep net. It's exactly the same as earlier, with just one addition: we add a `tf.summary.scalar()` to track the loss and the accuracy during training, so we can view nice learning curves using TensorBoard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "    loss_summary = tf.summary.scalar('log_loss', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to define the directory to write the TensorBoard logs to:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def log_dir(prefix=\"\"):\n",
    "    now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "    root_logdir = \"tf_logs\"\n",
    "    if prefix:\n",
    "        prefix += \"-\"\n",
    "    name = prefix + \"run-\" + now\n",
    "    return \"{}/{}/\".format(root_logdir, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdir = log_dir(\"mnist_dnn\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can create the `FileWriter` that we will use to write the TensorBoard logs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hey! Why don't we implement early stopping? For this, we are going to need to use the validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "m, n = X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 \tValidation accuracy: 92.180% \tLoss: 0.30208\n",
      "Epoch: 5 \tValidation accuracy: 95.980% \tLoss: 0.15037\n",
      "Epoch: 10 \tValidation accuracy: 97.100% \tLoss: 0.11160\n",
      "Epoch: 15 \tValidation accuracy: 97.700% \tLoss: 0.09562\n",
      "Epoch: 20 \tValidation accuracy: 97.840% \tLoss: 0.08309\n",
      "Epoch: 25 \tValidation accuracy: 98.040% \tLoss: 0.07706\n",
      "Epoch: 30 \tValidation accuracy: 98.140% \tLoss: 0.07287\n",
      "Epoch: 35 \tValidation accuracy: 98.280% \tLoss: 0.07133\n",
      "Epoch: 40 \tValidation accuracy: 98.220% \tLoss: 0.06968\n",
      "Epoch: 45 \tValidation accuracy: 98.220% \tLoss: 0.06993\n",
      "Epoch: 50 \tValidation accuracy: 98.160% \tLoss: 0.07093\n",
      "Epoch: 55 \tValidation accuracy: 98.280% \tLoss: 0.06994\n",
      "Epoch: 60 \tValidation accuracy: 98.200% \tLoss: 0.06894\n",
      "Epoch: 65 \tValidation accuracy: 98.260% \tLoss: 0.06906\n",
      "Epoch: 70 \tValidation accuracy: 98.220% \tLoss: 0.07057\n",
      "Epoch: 75 \tValidation accuracy: 98.280% \tLoss: 0.06963\n",
      "Epoch: 80 \tValidation accuracy: 98.320% \tLoss: 0.07264\n",
      "Epoch: 85 \tValidation accuracy: 98.200% \tLoss: 0.07403\n",
      "Epoch: 90 \tValidation accuracy: 98.300% \tLoss: 0.07332\n",
      "Epoch: 95 \tValidation accuracy: 98.180% \tLoss: 0.07535\n",
      "Epoch: 100 \tValidation accuracy: 98.260% \tLoss: 0.07542\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10001\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "checkpoint_path = \"/tmp/my_deep_mnist_model.ckpt\"\n",
    "checkpoint_epoch_path = checkpoint_path + \".epoch\"\n",
    "final_model_path = \"./my_deep_mnist_model\"\n",
    "\n",
    "best_loss = np.infty\n",
    "epochs_without_progress = 0\n",
    "max_epochs_without_progress = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if os.path.isfile(checkpoint_epoch_path):\n",
    "        # if the checkpoint file exists, restore the model and load the epoch number\n",
    "        with open(checkpoint_epoch_path, \"rb\") as f:\n",
    "            start_epoch = int(f.read())\n",
    "        print(\"Training was interrupted. Continuing at epoch\", start_epoch)\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        sess.run(init)\n",
    "\n",
    "    for epoch in range(start_epoch, n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})\n",
    "        file_writer.add_summary(accuracy_summary_str, epoch)\n",
    "        file_writer.add_summary(loss_summary_str, epoch)\n",
    "        if epoch % 5 == 0:\n",
    "            print(\"Epoch:\", epoch,\n",
    "                  \"\\tValidation accuracy: {:.3f}%\".format(accuracy_val * 100),\n",
    "                  \"\\tLoss: {:.5f}\".format(loss_val))\n",
    "            saver.save(sess, checkpoint_path)\n",
    "            with open(checkpoint_epoch_path, \"wb\") as f:\n",
    "                f.write(b\"%d\" % (epoch + 1))\n",
    "            if loss_val < best_loss:\n",
    "                saver.save(sess, final_model_path)\n",
    "                best_loss = loss_val\n",
    "            else:\n",
    "                epochs_without_progress += 5\n",
    "                if epochs_without_progress > max_epochs_without_progress:\n",
    "                    print(\"Early stopping\")\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(checkpoint_epoch_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, final_model_path)\n",
    "    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9796"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "nav_menu": {
   "height": "264px",
   "width": "369px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}