Project 1 - CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load Packages\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4546, 721)"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Load dataset and create a dataframe\n",
    "data_imgs = np.loadtxt('imgs-train.txt', delimiter = ' ')\n",
    "data_label = np.loadtxt('label-train.txt')\n",
    "data_label = data_label[:,1]\n",
    "data_imgs = data_imgs.astype(np.float32)\n",
    "data_label = data_label.astype(np.float32)\n",
    "df_imgs = pd.DataFrame(data_imgs)\n",
    "df_imgs\n",
    "df_label = pd.DataFrame(data_label)\n",
    "df_label\n",
    "df = pd.concat([df_imgs, df_label], axis=1)\n",
    "\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#Split dataset into train set and test set\n",
    "X = df.iloc[:,0:720]\n",
    "y = df.iloc[:,720]\n",
    "X_train_raw, X_test_raw, y_train_raw, y_test_raw = train_test_split(X, y, test_size = 0.33, random_state = 1992)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>710</th>\n",
       "      <th>711</th>\n",
       "      <th>712</th>\n",
       "      <th>713</th>\n",
       "      <th>714</th>\n",
       "      <th>715</th>\n",
       "      <th>716</th>\n",
       "      <th>717</th>\n",
       "      <th>718</th>\n",
       "      <th>719</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3966</th>\n",
       "      <td>1324.0</td>\n",
       "      <td>1284.0</td>\n",
       "      <td>1118.0</td>\n",
       "      <td>984.0</td>\n",
       "      <td>878.0</td>\n",
       "      <td>504.0</td>\n",
       "      <td>406.0</td>\n",
       "      <td>684.0</td>\n",
       "      <td>1082.0</td>\n",
       "      <td>1330.0</td>\n",
       "      <td>...</td>\n",
       "      <td>422.0</td>\n",
       "      <td>754.0</td>\n",
       "      <td>832.0</td>\n",
       "      <td>1142.0</td>\n",
       "      <td>958.0</td>\n",
       "      <td>1102.0</td>\n",
       "      <td>988.0</td>\n",
       "      <td>888.0</td>\n",
       "      <td>674.0</td>\n",
       "      <td>456.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2749</th>\n",
       "      <td>260.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>202.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>386.0</td>\n",
       "      <td>452.0</td>\n",
       "      <td>350.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>620.0</td>\n",
       "      <td>...</td>\n",
       "      <td>742.0</td>\n",
       "      <td>732.0</td>\n",
       "      <td>834.0</td>\n",
       "      <td>758.0</td>\n",
       "      <td>874.0</td>\n",
       "      <td>734.0</td>\n",
       "      <td>572.0</td>\n",
       "      <td>426.0</td>\n",
       "      <td>446.0</td>\n",
       "      <td>464.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2530</th>\n",
       "      <td>498.0</td>\n",
       "      <td>692.0</td>\n",
       "      <td>330.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>344.0</td>\n",
       "      <td>422.0</td>\n",
       "      <td>474.0</td>\n",
       "      <td>348.0</td>\n",
       "      <td>308.0</td>\n",
       "      <td>...</td>\n",
       "      <td>336.0</td>\n",
       "      <td>376.0</td>\n",
       "      <td>592.0</td>\n",
       "      <td>962.0</td>\n",
       "      <td>948.0</td>\n",
       "      <td>686.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>252.0</td>\n",
       "      <td>338.0</td>\n",
       "      <td>284.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3065</th>\n",
       "      <td>392.0</td>\n",
       "      <td>386.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>...</td>\n",
       "      <td>138.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>320.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>316.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1066</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>240.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>...</td>\n",
       "      <td>172.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>460.0</td>\n",
       "      <td>316.0</td>\n",
       "      <td>324.0</td>\n",
       "      <td>246.0</td>\n",
       "      <td>264.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 720 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2      3      4      5      6      7       8    \\\n",
       "3966  1324.0  1284.0  1118.0  984.0  878.0  504.0  406.0  684.0  1082.0   \n",
       "2749   260.0   190.0   202.0  132.0  260.0  386.0  452.0  350.0   360.0   \n",
       "2530   498.0   692.0   330.0  296.0  184.0  344.0  422.0  474.0   348.0   \n",
       "3065   392.0   386.0   144.0  120.0   78.0   98.0   80.0   70.0    22.0   \n",
       "1066     0.0     0.0    98.0  168.0  240.0  126.0   64.0   38.0    80.0   \n",
       "\n",
       "         9    ...      710    711    712     713    714     715    716    717  \\\n",
       "3966  1330.0  ...    422.0  754.0  832.0  1142.0  958.0  1102.0  988.0  888.0   \n",
       "2749   620.0  ...    742.0  732.0  834.0   758.0  874.0   734.0  572.0  426.0   \n",
       "2530   308.0  ...    336.0  376.0  592.0   962.0  948.0   686.0  360.0  252.0   \n",
       "3065    64.0  ...    138.0   94.0  176.0   296.0  320.0   212.0  142.0   98.0   \n",
       "1066    62.0  ...    172.0  276.0  460.0   316.0  324.0   246.0  264.0  128.0   \n",
       "\n",
       "        718    719  \n",
       "3966  674.0  456.0  \n",
       "2749  446.0  464.0  \n",
       "2530  338.0  284.0  \n",
       "3065  154.0  316.0  \n",
       "1066   48.0    0.0  \n",
       "\n",
       "[5 rows x 720 columns]"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3966    1.0\n",
       "2749    0.0\n",
       "2530    1.0\n",
       "3065    0.0\n",
       "1066    1.0\n",
       "Name: 0, dtype: float32"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transfer dataframe to array and create a better shape\n",
    "X_train = np.array(X_train_raw)\n",
    "X_train_image= tf.reshape(X_train, [-1,60,12,1])\n",
    "\n",
    "X_test = np.array(X_test_raw)\n",
    "X_test_image= tf.reshape(X_test, [-1,60,12,1])\n",
    "\n",
    "y_train = np.array(y_train_raw)\n",
    "y_train = y_train.astype(np.float32).reshape(-1,1)\n",
    "y_train = np.array(OneHotEncoder().fit_transform(y_train).todense())\n",
    "\n",
    "y_test = np.array(y_test_raw)\n",
    "y_test = y_test.astype(np.float32).reshape(-1,1)\n",
    "y_test = np.array(OneHotEncoder().fit_transform(y_test).todense())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  0.],\n",
       "       [ 0.,  1.],\n",
       "       [ 0.,  1.],\n",
       "       ..., \n",
       "       [ 1.,  0.],\n",
       "       [ 1.,  0.],\n",
       "       [ 1.,  0.]])"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3045, 720)"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x  = tf.placeholder(\"float\", shape=[None, 720])\n",
    "y_ = tf.placeholder(\"float\", shape=[None, 2])\n",
    "\n",
    "x_image = tf.reshape(x, [-1,60,12,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Define \n",
    "def weight_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(0.1, shape=shape)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "def conv2d(x, W):\n",
    "    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_conv1 = weight_variable([1, 1, 1, 8])\n",
    "b_conv1 = bias_variable([8])\n",
    "\n",
    "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "h_pool1 = max_pool_2x2(h_conv1)\n",
    "\n",
    "\n",
    "W_conv2 = weight_variable([4, 4, 8, 12])\n",
    "b_conv2 = bias_variable([12])\n",
    "\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "h_pool2 = max_pool_2x2(h_conv2)\n",
    "\n",
    "\n",
    "W_fc1 = weight_variable([15 * 3 * 12, 24])\n",
    "b_fc1 = bias_variable([24])\n",
    "\n",
    "h_pool2_flat = tf.reshape(h_pool2, [-1, 15 * 3 * 12])\n",
    "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
    "\n",
    "keep_prob = tf.placeholder(\"float\")\n",
    "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "W_fc2 = weight_variable([24, 2])\n",
    "b_fc2 = bias_variable([2])\n",
    "\n",
    "y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy = -tf.reduce_sum(y_*tf.log(y_conv))\n",
    "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0, training accuracy 0.75\n",
      "step 100, training accuracy 0.7\n",
      "step 200, training accuracy 0.7\n",
      "step 300, training accuracy 0.75\n",
      "step 400, training accuracy 0.8\n",
      "step 500, training accuracy 0.8\n",
      "step 600, training accuracy 0.6\n",
      "step 700, training accuracy 0.65\n",
      "step 800, training accuracy 0.65\n",
      "step 900, training accuracy 0.5\n",
      "test accuracy 0.8\n"
     ]
    }
   ],
   "source": [
    "#Run and check the accuracy\n",
    "batch_size = 20\n",
    "\n",
    "for i in range(1000):\n",
    "    indices_train = np.random.choice(X_train.shape[0], batch_size)\n",
    "    images_train_batch = X_train[indices_train]\n",
    "    labels_train_batch = y_train[indices_train]\n",
    "    \n",
    "    indices_test = np.random.choice(X_test.shape[0], batch_size)\n",
    "    images_test_batch = X_test[indices_test]\n",
    "    labels_test_batch = y_test[indices_test]\n",
    "    if i%100 == 0:\n",
    "        train_accuracy = sess.run(accuracy, feed_dict={x: images_train_batch,\n",
    "                                                           y_: labels_train_batch, keep_prob: 1.0})\n",
    "        print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "\n",
    "    sess.run(train_step,feed_dict={x: images_test_batch,y_: labels_test_batch, keep_prob: 0.5})\n",
    "\n",
    "print(\"test accuracy %g\"% sess.run(accuracy, feed_dict={x: images_test_batch,y_: labels_test_batch, keep_prob: 1.0}))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}