4-Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4-Model the Data \n",
    "\n",
    "> \"All models are wrong, Some of them are useful\"\n",
    "\n",
    "- The power and limits of models\n",
    "- Tradeoff between Prediction Accuracy and Model Interpretability\n",
    "- Assessing Model Accuracy\n",
    "- Regression models (Simple, Multiple)\n",
    "- Classification model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load the libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import stats\n",
    "from sklearn import linear_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(449, 7)\n",
      "(532, 7)\n"
     ]
    }
   ],
   "source": [
    "# Load the data again!\n",
    "df = pd.read_csv(\"data/Weed_Price.csv\", parse_dates=[-1])\n",
    "df.sort(columns=['State','date'], inplace=True)\n",
    "df1 = df[df.State==\"California\"].copy()\n",
    "df1.set_index(\"date\", inplace=True)\n",
    "print df1.shape\n",
    "idx = pd.date_range(df1.index.min(), df1.index.max())\n",
    "df1 = df1.reindex(idx)\n",
    "df1.fillna(method = \"ffill\", inplace=True)\n",
    "print df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>HighQ</th>\n",
       "      <th>HighQN</th>\n",
       "      <th>MedQ</th>\n",
       "      <th>MedQN</th>\n",
       "      <th>LowQ</th>\n",
       "      <th>LowQN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2013-12-27</th>\n",
       "      <td>California</td>\n",
       "      <td>248.77</td>\n",
       "      <td>12021</td>\n",
       "      <td>193.44</td>\n",
       "      <td>12724</td>\n",
       "      <td>193.88</td>\n",
       "      <td>770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-12-28</th>\n",
       "      <td>California</td>\n",
       "      <td>248.74</td>\n",
       "      <td>12025</td>\n",
       "      <td>193.44</td>\n",
       "      <td>12728</td>\n",
       "      <td>193.88</td>\n",
       "      <td>770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-12-29</th>\n",
       "      <td>California</td>\n",
       "      <td>248.76</td>\n",
       "      <td>12047</td>\n",
       "      <td>193.55</td>\n",
       "      <td>12760</td>\n",
       "      <td>193.60</td>\n",
       "      <td>772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-12-30</th>\n",
       "      <td>California</td>\n",
       "      <td>248.82</td>\n",
       "      <td>12065</td>\n",
       "      <td>193.54</td>\n",
       "      <td>12779</td>\n",
       "      <td>193.80</td>\n",
       "      <td>773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-12-31</th>\n",
       "      <td>California</td>\n",
       "      <td>248.76</td>\n",
       "      <td>12082</td>\n",
       "      <td>193.54</td>\n",
       "      <td>12792</td>\n",
       "      <td>193.80</td>\n",
       "      <td>773</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 State   HighQ  HighQN    MedQ  MedQN    LowQ  LowQN\n",
       "2013-12-27  California  248.77   12021  193.44  12724  193.88    770\n",
       "2013-12-28  California  248.74   12025  193.44  12728  193.88    770\n",
       "2013-12-29  California  248.76   12047  193.55  12760  193.60    772\n",
       "2013-12-30  California  248.82   12065  193.54  12779  193.80    773\n",
       "2013-12-31  California  248.76   12082  193.54  12792  193.80    773"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Reading demographics data\n",
    "demographics = pd.DataFrame.from_csv(\"data/Demographics_State.csv\",header=0,index_col=False,sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>total_population</th>\n",
       "      <th>percent_white</th>\n",
       "      <th>percent_black</th>\n",
       "      <th>percent_asian</th>\n",
       "      <th>percent_hispanic</th>\n",
       "      <th>per_capita_income</th>\n",
       "      <th>median_rent</th>\n",
       "      <th>median_age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>alabama</td>\n",
       "      <td>4799277</td>\n",
       "      <td>67</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>23680</td>\n",
       "      <td>501</td>\n",
       "      <td>38.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>alaska</td>\n",
       "      <td>720316</td>\n",
       "      <td>63</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>32651</td>\n",
       "      <td>978</td>\n",
       "      <td>33.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>arizona</td>\n",
       "      <td>6479703</td>\n",
       "      <td>57</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>30</td>\n",
       "      <td>25358</td>\n",
       "      <td>747</td>\n",
       "      <td>36.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>arkansas</td>\n",
       "      <td>2933369</td>\n",
       "      <td>74</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>22170</td>\n",
       "      <td>480</td>\n",
       "      <td>37.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>california</td>\n",
       "      <td>37659181</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>38</td>\n",
       "      <td>29527</td>\n",
       "      <td>1119</td>\n",
       "      <td>35.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        State  total_population  percent_white  percent_black  percent_asian  \\\n",
       "0     alabama           4799277             67             26              1   \n",
       "1      alaska            720316             63              3              5   \n",
       "2     arizona           6479703             57              4              3   \n",
       "3    arkansas           2933369             74             15              1   \n",
       "4  california          37659181             40              6             13   \n",
       "\n",
       "   percent_hispanic  per_capita_income  median_rent  median_age  \n",
       "0                 4              23680          501        38.1  \n",
       "1                 6              32651          978        33.6  \n",
       "2                30              25358          747        36.3  \n",
       "3                 7              22170          480        37.5  \n",
       "4                38              29527         1119        35.4  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "demographics.rename(columns={'region':'State'}, inplace=True)\n",
    "demographics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>HighQ</th>\n",
       "      <th>HighQN</th>\n",
       "      <th>MedQ</th>\n",
       "      <th>MedQN</th>\n",
       "      <th>LowQ</th>\n",
       "      <th>LowQN</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20094</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.65</td>\n",
       "      <td>1033</td>\n",
       "      <td>198.04</td>\n",
       "      <td>926</td>\n",
       "      <td>147.15</td>\n",
       "      <td>122</td>\n",
       "      <td>2013-12-27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20859</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.65</td>\n",
       "      <td>1033</td>\n",
       "      <td>198.04</td>\n",
       "      <td>926</td>\n",
       "      <td>147.15</td>\n",
       "      <td>122</td>\n",
       "      <td>2013-12-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21573</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.75</td>\n",
       "      <td>1036</td>\n",
       "      <td>198.26</td>\n",
       "      <td>929</td>\n",
       "      <td>149.49</td>\n",
       "      <td>123</td>\n",
       "      <td>2013-12-29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22287</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.75</td>\n",
       "      <td>1036</td>\n",
       "      <td>198.81</td>\n",
       "      <td>930</td>\n",
       "      <td>149.49</td>\n",
       "      <td>123</td>\n",
       "      <td>2013-12-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22797</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.42</td>\n",
       "      <td>1040</td>\n",
       "      <td>198.68</td>\n",
       "      <td>932</td>\n",
       "      <td>149.49</td>\n",
       "      <td>123</td>\n",
       "      <td>2013-12-31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         State   HighQ  HighQN    MedQ  MedQN    LowQ  LowQN       date\n",
       "20094  alabama  339.65    1033  198.04    926  147.15    122 2013-12-27\n",
       "20859  alabama  339.65    1033  198.04    926  147.15    122 2013-12-28\n",
       "21573  alabama  339.75    1036  198.26    929  149.49    123 2013-12-29\n",
       "22287  alabama  339.75    1036  198.81    930  149.49    123 2013-12-30\n",
       "22797  alabama  339.42    1040  198.68    932  149.49    123 2013-12-31"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['State'] = df['State'].str.lower()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>HighQ</th>\n",
       "      <th>HighQN</th>\n",
       "      <th>MedQ</th>\n",
       "      <th>MedQN</th>\n",
       "      <th>LowQ</th>\n",
       "      <th>LowQN</th>\n",
       "      <th>date</th>\n",
       "      <th>total_population</th>\n",
       "      <th>percent_white</th>\n",
       "      <th>percent_black</th>\n",
       "      <th>percent_asian</th>\n",
       "      <th>percent_hispanic</th>\n",
       "      <th>per_capita_income</th>\n",
       "      <th>median_rent</th>\n",
       "      <th>median_age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.65</td>\n",
       "      <td>1033</td>\n",
       "      <td>198.04</td>\n",
       "      <td>926</td>\n",
       "      <td>147.15</td>\n",
       "      <td>122</td>\n",
       "      <td>2013-12-27</td>\n",
       "      <td>4799277</td>\n",
       "      <td>67</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>23680</td>\n",
       "      <td>501</td>\n",
       "      <td>38.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.65</td>\n",
       "      <td>1033</td>\n",
       "      <td>198.04</td>\n",
       "      <td>926</td>\n",
       "      <td>147.15</td>\n",
       "      <td>122</td>\n",
       "      <td>2013-12-28</td>\n",
       "      <td>4799277</td>\n",
       "      <td>67</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>23680</td>\n",
       "      <td>501</td>\n",
       "      <td>38.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.75</td>\n",
       "      <td>1036</td>\n",
       "      <td>198.26</td>\n",
       "      <td>929</td>\n",
       "      <td>149.49</td>\n",
       "      <td>123</td>\n",
       "      <td>2013-12-29</td>\n",
       "      <td>4799277</td>\n",
       "      <td>67</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>23680</td>\n",
       "      <td>501</td>\n",
       "      <td>38.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.75</td>\n",
       "      <td>1036</td>\n",
       "      <td>198.81</td>\n",
       "      <td>930</td>\n",
       "      <td>149.49</td>\n",
       "      <td>123</td>\n",
       "      <td>2013-12-30</td>\n",
       "      <td>4799277</td>\n",
       "      <td>67</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>23680</td>\n",
       "      <td>501</td>\n",
       "      <td>38.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>alabama</td>\n",
       "      <td>339.42</td>\n",
       "      <td>1040</td>\n",
       "      <td>198.68</td>\n",
       "      <td>932</td>\n",
       "      <td>149.49</td>\n",
       "      <td>123</td>\n",
       "      <td>2013-12-31</td>\n",
       "      <td>4799277</td>\n",
       "      <td>67</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>23680</td>\n",
       "      <td>501</td>\n",
       "      <td>38.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     State   HighQ  HighQN    MedQ  MedQN    LowQ  LowQN       date  \\\n",
       "0  alabama  339.65    1033  198.04    926  147.15    122 2013-12-27   \n",
       "1  alabama  339.65    1033  198.04    926  147.15    122 2013-12-28   \n",
       "2  alabama  339.75    1036  198.26    929  149.49    123 2013-12-29   \n",
       "3  alabama  339.75    1036  198.81    930  149.49    123 2013-12-30   \n",
       "4  alabama  339.42    1040  198.68    932  149.49    123 2013-12-31   \n",
       "\n",
       "   total_population  percent_white  percent_black  percent_asian  \\\n",
       "0           4799277             67             26              1   \n",
       "1           4799277             67             26              1   \n",
       "2           4799277             67             26              1   \n",
       "3           4799277             67             26              1   \n",
       "4           4799277             67             26              1   \n",
       "\n",
       "   percent_hispanic  per_capita_income  median_rent  median_age  \n",
       "0                 4              23680          501        38.1  \n",
       "1                 4              23680          501        38.1  \n",
       "2                 4              23680          501        38.1  \n",
       "3                 4              23680          501        38.1  \n",
       "4                 4              23680          501        38.1  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_demo = pd.merge(df, demographics, how=\"inner\", on=\"State\")\n",
    "df_demo.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.120703416389\n"
     ]
    }
   ],
   "source": [
    "corr_bw_percapita_highq = stats.pearsonr(df_demo.per_capita_income, df_demo.HighQ)[0]\n",
    "print corr_bw_percapita_highq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise** Find correlation between percent_white and highQ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Impact of de-regulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>region</th>\n",
       "      <th>state</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>alaska</td>\n",
       "      <td>AK</td>\n",
       "      <td>61.3850</td>\n",
       "      <td>-152.2683</td>\n",
       "      <td>legal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>alabama</td>\n",
       "      <td>AL</td>\n",
       "      <td>32.7990</td>\n",
       "      <td>-86.8073</td>\n",
       "      <td>medical-limited</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>arkansas</td>\n",
       "      <td>AR</td>\n",
       "      <td>34.9513</td>\n",
       "      <td>-92.3809</td>\n",
       "      <td>illegal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>arizona</td>\n",
       "      <td>AZ</td>\n",
       "      <td>33.7712</td>\n",
       "      <td>-111.3877</td>\n",
       "      <td>medical</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>california</td>\n",
       "      <td>CA</td>\n",
       "      <td>36.1700</td>\n",
       "      <td>-119.7462</td>\n",
       "      <td>decriminalized+medical</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       region state  latitude  longitude                  status\n",
       "0      alaska    AK   61.3850  -152.2683                   legal\n",
       "1     alabama    AL   32.7990   -86.8073         medical-limited\n",
       "2    arkansas    AR   34.9513   -92.3809                 illegal\n",
       "3     arizona    AZ   33.7712  -111.3877                 medical\n",
       "4  california    CA   36.1700  -119.7462  decriminalized+medical"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_location = pd.read_csv(\"data/State_Location.csv\")\n",
    "state_location.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['legal', 'medical-limited', 'illegal', 'medical',\n",
       "       'decriminalized+medical', 'decriminalized'], dtype=object)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.unique(state_location.status)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise** Find mean prices of HighQ weed for states that are legal and for states that are illegal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding good time of the week to buy weed in California"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['year'] = pd.DatetimeIndex(df['date']).year\n",
    "df['month'] = pd.DatetimeIndex(df['date']).month\n",
    "df['week'] = pd.DatetimeIndex(df['date']).week\n",
    "df['weekday'] = pd.DatetimeIndex(df['date']).weekday"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>HighQ</th>\n",
       "      <th>HighQN</th>\n",
       "      <th>MedQ</th>\n",
       "      <th>MedQN</th>\n",
       "      <th>LowQ</th>\n",
       "      <th>LowQN</th>\n",
       "      <th>date</th>\n",
       "      <th>total_population</th>\n",
       "      <th>percent_white</th>\n",
       "      <th>percent_black</th>\n",
       "      <th>percent_asian</th>\n",
       "      <th>percent_hispanic</th>\n",
       "      <th>per_capita_income</th>\n",
       "      <th>median_rent</th>\n",
       "      <th>median_age</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>week</th>\n",
       "      <th>weekday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1796</th>\n",
       "      <td>california</td>\n",
       "      <td>248.77</td>\n",
       "      <td>12021</td>\n",
       "      <td>193.44</td>\n",
       "      <td>12724</td>\n",
       "      <td>193.88</td>\n",
       "      <td>770</td>\n",
       "      <td>2013-12-27</td>\n",
       "      <td>37659181</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>38</td>\n",
       "      <td>29527</td>\n",
       "      <td>1119</td>\n",
       "      <td>35.4</td>\n",
       "      <td>2013</td>\n",
       "      <td>12</td>\n",
       "      <td>52</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1797</th>\n",
       "      <td>california</td>\n",
       "      <td>248.74</td>\n",
       "      <td>12025</td>\n",
       "      <td>193.44</td>\n",
       "      <td>12728</td>\n",
       "      <td>193.88</td>\n",
       "      <td>770</td>\n",
       "      <td>2013-12-28</td>\n",
       "      <td>37659181</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>38</td>\n",
       "      <td>29527</td>\n",
       "      <td>1119</td>\n",
       "      <td>35.4</td>\n",
       "      <td>2013</td>\n",
       "      <td>12</td>\n",
       "      <td>52</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1798</th>\n",
       "      <td>california</td>\n",
       "      <td>248.76</td>\n",
       "      <td>12047</td>\n",
       "      <td>193.55</td>\n",
       "      <td>12760</td>\n",
       "      <td>193.60</td>\n",
       "      <td>772</td>\n",
       "      <td>2013-12-29</td>\n",
       "      <td>37659181</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>38</td>\n",
       "      <td>29527</td>\n",
       "      <td>1119</td>\n",
       "      <td>35.4</td>\n",
       "      <td>2013</td>\n",
       "      <td>12</td>\n",
       "      <td>52</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1799</th>\n",
       "      <td>california</td>\n",
       "      <td>248.82</td>\n",
       "      <td>12065</td>\n",
       "      <td>193.54</td>\n",
       "      <td>12779</td>\n",
       "      <td>193.80</td>\n",
       "      <td>773</td>\n",
       "      <td>2013-12-30</td>\n",
       "      <td>37659181</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>38</td>\n",
       "      <td>29527</td>\n",
       "      <td>1119</td>\n",
       "      <td>35.4</td>\n",
       "      <td>2013</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1800</th>\n",
       "      <td>california</td>\n",
       "      <td>248.76</td>\n",
       "      <td>12082</td>\n",
       "      <td>193.54</td>\n",
       "      <td>12792</td>\n",
       "      <td>193.80</td>\n",
       "      <td>773</td>\n",
       "      <td>2013-12-31</td>\n",
       "      <td>37659181</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>38</td>\n",
       "      <td>29527</td>\n",
       "      <td>1119</td>\n",
       "      <td>35.4</td>\n",
       "      <td>2013</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           State   HighQ  HighQN    MedQ  MedQN    LowQ  LowQN       date  \\\n",
       "1796  california  248.77   12021  193.44  12724  193.88    770 2013-12-27   \n",
       "1797  california  248.74   12025  193.44  12728  193.88    770 2013-12-28   \n",
       "1798  california  248.76   12047  193.55  12760  193.60    772 2013-12-29   \n",
       "1799  california  248.82   12065  193.54  12779  193.80    773 2013-12-30   \n",
       "1800  california  248.76   12082  193.54  12792  193.80    773 2013-12-31   \n",
       "\n",
       "      total_population  percent_white  percent_black  percent_asian  \\\n",
       "1796          37659181             40              6             13   \n",
       "1797          37659181             40              6             13   \n",
       "1798          37659181             40              6             13   \n",
       "1799          37659181             40              6             13   \n",
       "1800          37659181             40              6             13   \n",
       "\n",
       "      percent_hispanic  per_capita_income  median_rent  median_age  year  \\\n",
       "1796                38              29527         1119        35.4  2013   \n",
       "1797                38              29527         1119        35.4  2013   \n",
       "1798                38              29527         1119        35.4  2013   \n",
       "1799                38              29527         1119        35.4  2013   \n",
       "1800                38              29527         1119        35.4  2013   \n",
       "\n",
       "      month  week  weekday  \n",
       "1796     12    52        4  \n",
       "1797     12    52        5  \n",
       "1798     12    52        6  \n",
       "1799     12     1        0  \n",
       "1800     12     1        1  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_demo_ca = df_demo[df_demo.State==\"california\"].copy()\n",
    "df_demo_ca['year'] = pd.DatetimeIndex(df_demo_ca['date']).year\n",
    "df_demo_ca['month'] = pd.DatetimeIndex(df_demo_ca['date']).month\n",
    "df_demo_ca['week'] = pd.DatetimeIndex(df_demo_ca['date']).week\n",
    "df_demo_ca['weekday'] = pd.DatetimeIndex(df_demo_ca['date']).weekday\n",
    "\n",
    "df_demo_ca.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "weekday\n",
       "0    245.348923\n",
       "1    245.364375\n",
       "2    245.359219\n",
       "3    245.342813\n",
       "4    245.434219\n",
       "5    245.425556\n",
       "6    245.359231\n",
       "Name: HighQ, dtype: float64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_demo_ca.groupby(\"weekday\").HighQ.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise** If I need to buy weed on a wednesday, which state should I be in? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "State          weekday\n",
       "Alabama        0          339.556308\n",
       "               1          339.577656\n",
       "               2          339.559375\n",
       "               3          339.552500\n",
       "               4          339.577969\n",
       "               5          339.568254\n",
       "               6          339.541385\n",
       "Alaska         0          291.463231\n",
       "               1          291.384687\n",
       "               2          291.390000\n",
       "               3          291.597500\n",
       "               4          291.506406\n",
       "               5          291.596032\n",
       "               6          291.438923\n",
       "Arizona        0          300.612000\n",
       "               1          300.653906\n",
       "               2          300.643750\n",
       "               3          300.655000\n",
       "               4          300.751562\n",
       "               5          300.715873\n",
       "               6          300.642308\n",
       "Arkansas       0          347.930462\n",
       "               1          348.011250\n",
       "               2          348.017656\n",
       "               3          347.934844\n",
       "               4          348.278281\n",
       "               5          348.235238\n",
       "               6          347.991077\n",
       "California     0          245.348923\n",
       "               1          245.364375\n",
       "                             ...    \n",
       "Virginia       5          368.245238\n",
       "               6          368.116462\n",
       "Washington     0          233.381231\n",
       "               1          233.377969\n",
       "               2          233.361875\n",
       "               3          233.363594\n",
       "               4          233.422500\n",
       "               5          233.426032\n",
       "               6          233.382308\n",
       "West Virginia  0          357.408000\n",
       "               1          357.428906\n",
       "               2          357.368125\n",
       "               3          357.330156\n",
       "               4          357.395938\n",
       "               5          357.424444\n",
       "               6          357.406462\n",
       "Wisconsin      0          352.981077\n",
       "               1          353.050313\n",
       "               2          353.040469\n",
       "               3          353.000938\n",
       "               4          353.160000\n",
       "               5          353.141905\n",
       "               6          352.996308\n",
       "Wyoming        0          338.837538\n",
       "               1          339.090625\n",
       "               2          339.045781\n",
       "               3          338.932969\n",
       "               4          339.260469\n",
       "               5          339.072698\n",
       "               6          338.910769\n",
       "Name: HighQ, dtype: float64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby([\"State\", \"weekday\"]).HighQ.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_st_wk = df.groupby([\"State\", \"weekday\"]).HighQ.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>weekday</th>\n",
       "      <th>HighQ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alabama</td>\n",
       "      <td>0</td>\n",
       "      <td>339.556308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alabama</td>\n",
       "      <td>1</td>\n",
       "      <td>339.577656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Alabama</td>\n",
       "      <td>2</td>\n",
       "      <td>339.559375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Alabama</td>\n",
       "      <td>3</td>\n",
       "      <td>339.552500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alabama</td>\n",
       "      <td>4</td>\n",
       "      <td>339.577969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Alabama</td>\n",
       "      <td>5</td>\n",
       "      <td>339.568254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Alabama</td>\n",
       "      <td>6</td>\n",
       "      <td>339.541385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>0</td>\n",
       "      <td>291.463231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>1</td>\n",
       "      <td>291.384687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>2</td>\n",
       "      <td>291.390000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>3</td>\n",
       "      <td>291.597500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>4</td>\n",
       "      <td>291.506406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>5</td>\n",
       "      <td>291.596032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Alaska</td>\n",
       "      <td>6</td>\n",
       "      <td>291.438923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Arizona</td>\n",
       "      <td>0</td>\n",
       "      <td>300.612000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Arizona</td>\n",
       "      <td>1</td>\n",
       "      <td>300.653906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Arizona</td>\n",
       "      <td>2</td>\n",
       "      <td>300.643750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Arizona</td>\n",
       "      <td>3</td>\n",
       "      <td>300.655000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Arizona</td>\n",
       "      <td>4</td>\n",
       "      <td>300.751562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Arizona</td>\n",
       "      <td>5</td>\n",
       "      <td>300.715873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Arizona</td>\n",
       "      <td>6</td>\n",
       "      <td>300.642308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Arkansas</td>\n",
       "      <td>0</td>\n",
       "      <td>347.930462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Arkansas</td>\n",
       "      <td>1</td>\n",
       "      <td>348.011250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Arkansas</td>\n",
       "      <td>2</td>\n",
       "      <td>348.017656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Arkansas</td>\n",
       "      <td>3</td>\n",
       "      <td>347.934844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Arkansas</td>\n",
       "      <td>4</td>\n",
       "      <td>348.278281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Arkansas</td>\n",
       "      <td>5</td>\n",
       "      <td>348.235238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Arkansas</td>\n",
       "      <td>6</td>\n",
       "      <td>347.991077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>California</td>\n",
       "      <td>0</td>\n",
       "      <td>245.348923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>California</td>\n",
       "      <td>1</td>\n",
       "      <td>245.364375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327</th>\n",
       "      <td>Virginia</td>\n",
       "      <td>5</td>\n",
       "      <td>368.245238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>328</th>\n",
       "      <td>Virginia</td>\n",
       "      <td>6</td>\n",
       "      <td>368.116462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>Washington</td>\n",
       "      <td>0</td>\n",
       "      <td>233.381231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>Washington</td>\n",
       "      <td>1</td>\n",
       "      <td>233.377969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>Washington</td>\n",
       "      <td>2</td>\n",
       "      <td>233.361875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>Washington</td>\n",
       "      <td>3</td>\n",
       "      <td>233.363594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>Washington</td>\n",
       "      <td>4</td>\n",
       "      <td>233.422500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>Washington</td>\n",
       "      <td>5</td>\n",
       "      <td>233.426032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>Washington</td>\n",
       "      <td>6</td>\n",
       "      <td>233.382308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>West Virginia</td>\n",
       "      <td>0</td>\n",
       "      <td>357.408000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>West Virginia</td>\n",
       "      <td>1</td>\n",
       "      <td>357.428906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>West Virginia</td>\n",
       "      <td>2</td>\n",
       "      <td>357.368125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>West Virginia</td>\n",
       "      <td>3</td>\n",
       "      <td>357.330156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>340</th>\n",
       "      <td>West Virginia</td>\n",
       "      <td>4</td>\n",
       "      <td>357.395938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341</th>\n",
       "      <td>West Virginia</td>\n",
       "      <td>5</td>\n",
       "      <td>357.424444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>West Virginia</td>\n",
       "      <td>6</td>\n",
       "      <td>357.406462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343</th>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>0</td>\n",
       "      <td>352.981077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344</th>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>1</td>\n",
       "      <td>353.050313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345</th>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>2</td>\n",
       "      <td>353.040469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346</th>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>3</td>\n",
       "      <td>353.000938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>347</th>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>4</td>\n",
       "      <td>353.160000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348</th>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>5</td>\n",
       "      <td>353.141905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>Wisconsin</td>\n",
       "      <td>6</td>\n",
       "      <td>352.996308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350</th>\n",
       "      <td>Wyoming</td>\n",
       "      <td>0</td>\n",
       "      <td>338.837538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351</th>\n",
       "      <td>Wyoming</td>\n",
       "      <td>1</td>\n",
       "      <td>339.090625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>Wyoming</td>\n",
       "      <td>2</td>\n",
       "      <td>339.045781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>Wyoming</td>\n",
       "      <td>3</td>\n",
       "      <td>338.932969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>354</th>\n",
       "      <td>Wyoming</td>\n",
       "      <td>4</td>\n",
       "      <td>339.260469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>Wyoming</td>\n",
       "      <td>5</td>\n",
       "      <td>339.072698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356</th>\n",
       "      <td>Wyoming</td>\n",
       "      <td>6</td>\n",
       "      <td>338.910769</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>357 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             State  weekday       HighQ\n",
       "0          Alabama        0  339.556308\n",
       "1          Alabama        1  339.577656\n",
       "2          Alabama        2  339.559375\n",
       "3          Alabama        3  339.552500\n",
       "4          Alabama        4  339.577969\n",
       "5          Alabama        5  339.568254\n",
       "6          Alabama        6  339.541385\n",
       "7           Alaska        0  291.463231\n",
       "8           Alaska        1  291.384687\n",
       "9           Alaska        2  291.390000\n",
       "10          Alaska        3  291.597500\n",
       "11          Alaska        4  291.506406\n",
       "12          Alaska        5  291.596032\n",
       "13          Alaska        6  291.438923\n",
       "14         Arizona        0  300.612000\n",
       "15         Arizona        1  300.653906\n",
       "16         Arizona        2  300.643750\n",
       "17         Arizona        3  300.655000\n",
       "18         Arizona        4  300.751562\n",
       "19         Arizona        5  300.715873\n",
       "20         Arizona        6  300.642308\n",
       "21        Arkansas        0  347.930462\n",
       "22        Arkansas        1  348.011250\n",
       "23        Arkansas        2  348.017656\n",
       "24        Arkansas        3  347.934844\n",
       "25        Arkansas        4  348.278281\n",
       "26        Arkansas        5  348.235238\n",
       "27        Arkansas        6  347.991077\n",
       "28      California        0  245.348923\n",
       "29      California        1  245.364375\n",
       "..             ...      ...         ...\n",
       "327       Virginia        5  368.245238\n",
       "328       Virginia        6  368.116462\n",
       "329     Washington        0  233.381231\n",
       "330     Washington        1  233.377969\n",
       "331     Washington        2  233.361875\n",
       "332     Washington        3  233.363594\n",
       "333     Washington        4  233.422500\n",
       "334     Washington        5  233.426032\n",
       "335     Washington        6  233.382308\n",
       "336  West Virginia        0  357.408000\n",
       "337  West Virginia        1  357.428906\n",
       "338  West Virginia        2  357.368125\n",
       "339  West Virginia        3  357.330156\n",
       "340  West Virginia        4  357.395938\n",
       "341  West Virginia        5  357.424444\n",
       "342  West Virginia        6  357.406462\n",
       "343      Wisconsin        0  352.981077\n",
       "344      Wisconsin        1  353.050313\n",
       "345      Wisconsin        2  353.040469\n",
       "346      Wisconsin        3  353.000938\n",
       "347      Wisconsin        4  353.160000\n",
       "348      Wisconsin        5  353.141905\n",
       "349      Wisconsin        6  352.996308\n",
       "350        Wyoming        0  338.837538\n",
       "351        Wyoming        1  339.090625\n",
       "352        Wyoming        2  339.045781\n",
       "353        Wyoming        3  338.932969\n",
       "354        Wyoming        4  339.260469\n",
       "355        Wyoming        5  339.072698\n",
       "356        Wyoming        6  338.910769\n",
       "\n",
       "[357 rows x 3 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_st_wk.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Answer:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predicting price of HighQ weed in CA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HighQ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-06-07</th>\n",
       "      <td>241.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-08</th>\n",
       "      <td>241.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-09</th>\n",
       "      <td>241.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-10</th>\n",
       "      <td>241.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-11</th>\n",
       "      <td>241.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-12</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-13</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-14</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-15</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-16</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-17</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-18</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-19</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-20</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-21</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-22</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-23</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-24</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-25</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-26</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-27</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-28</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-29</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-30</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-01</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-02</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-03</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-04</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-05</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-06</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-07</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-08</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-09</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-10</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-11</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             HighQ\n",
       "2015-06-07  241.91\n",
       "2015-06-08  241.93\n",
       "2015-06-09  241.93\n",
       "2015-06-10  241.93\n",
       "2015-06-11  241.93\n",
       "2015-06-12     NaN\n",
       "2015-06-13     NaN\n",
       "2015-06-14     NaN\n",
       "2015-06-15     NaN\n",
       "2015-06-16     NaN\n",
       "2015-06-17     NaN\n",
       "2015-06-18     NaN\n",
       "2015-06-19     NaN\n",
       "2015-06-20     NaN\n",
       "2015-06-21     NaN\n",
       "2015-06-22     NaN\n",
       "2015-06-23     NaN\n",
       "2015-06-24     NaN\n",
       "2015-06-25     NaN\n",
       "2015-06-26     NaN\n",
       "2015-06-27     NaN\n",
       "2015-06-28     NaN\n",
       "2015-06-29     NaN\n",
       "2015-06-30     NaN\n",
       "2015-07-01     NaN\n",
       "2015-07-02     NaN\n",
       "2015-07-03     NaN\n",
       "2015-07-04     NaN\n",
       "2015-07-05     NaN\n",
       "2015-07-06     NaN\n",
       "2015-07-07     NaN\n",
       "2015-07-08     NaN\n",
       "2015-07-09     NaN\n",
       "2015-07-10     NaN\n",
       "2015-07-11     NaN"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_data = df1.loc[:,['HighQ']].copy()\n",
    "idx = pd.date_range(model_data.index.min(), model_data.index.max()+ 30)\n",
    "model_data.reset_index(inplace=True)\n",
    "model_data.set_index(\"index\", inplace=True)\n",
    "model_data = model_data.reindex(idx)\n",
    "model_data.tail(35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HighQ</th>\n",
       "      <th>IND</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-06-07</th>\n",
       "      <td>241.91</td>\n",
       "      <td>527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-08</th>\n",
       "      <td>241.93</td>\n",
       "      <td>528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-09</th>\n",
       "      <td>241.93</td>\n",
       "      <td>529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-10</th>\n",
       "      <td>241.93</td>\n",
       "      <td>530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-11</th>\n",
       "      <td>241.93</td>\n",
       "      <td>531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-17</th>\n",
       "      <td>NaN</td>\n",
       "      <td>537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-19</th>\n",
       "      <td>NaN</td>\n",
       "      <td>539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-20</th>\n",
       "      <td>NaN</td>\n",
       "      <td>540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-21</th>\n",
       "      <td>NaN</td>\n",
       "      <td>541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-22</th>\n",
       "      <td>NaN</td>\n",
       "      <td>542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-23</th>\n",
       "      <td>NaN</td>\n",
       "      <td>543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-24</th>\n",
       "      <td>NaN</td>\n",
       "      <td>544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-25</th>\n",
       "      <td>NaN</td>\n",
       "      <td>545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-26</th>\n",
       "      <td>NaN</td>\n",
       "      <td>546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-27</th>\n",
       "      <td>NaN</td>\n",
       "      <td>547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-28</th>\n",
       "      <td>NaN</td>\n",
       "      <td>548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-29</th>\n",
       "      <td>NaN</td>\n",
       "      <td>549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-06-30</th>\n",
       "      <td>NaN</td>\n",
       "      <td>550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-02</th>\n",
       "      <td>NaN</td>\n",
       "      <td>552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-03</th>\n",
       "      <td>NaN</td>\n",
       "      <td>553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-04</th>\n",
       "      <td>NaN</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-05</th>\n",
       "      <td>NaN</td>\n",
       "      <td>555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-06</th>\n",
       "      <td>NaN</td>\n",
       "      <td>556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-07</th>\n",
       "      <td>NaN</td>\n",
       "      <td>557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-08</th>\n",
       "      <td>NaN</td>\n",
       "      <td>558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-09</th>\n",
       "      <td>NaN</td>\n",
       "      <td>559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-07-11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>561</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             HighQ  IND\n",
       "2015-06-07  241.91  527\n",
       "2015-06-08  241.93  528\n",
       "2015-06-09  241.93  529\n",
       "2015-06-10  241.93  530\n",
       "2015-06-11  241.93  531\n",
       "2015-06-12     NaN  532\n",
       "2015-06-13     NaN  533\n",
       "2015-06-14     NaN  534\n",
       "2015-06-15     NaN  535\n",
       "2015-06-16     NaN  536\n",
       "2015-06-17     NaN  537\n",
       "2015-06-18     NaN  538\n",
       "2015-06-19     NaN  539\n",
       "2015-06-20     NaN  540\n",
       "2015-06-21     NaN  541\n",
       "2015-06-22     NaN  542\n",
       "2015-06-23     NaN  543\n",
       "2015-06-24     NaN  544\n",
       "2015-06-25     NaN  545\n",
       "2015-06-26     NaN  546\n",
       "2015-06-27     NaN  547\n",
       "2015-06-28     NaN  548\n",
       "2015-06-29     NaN  549\n",
       "2015-06-30     NaN  550\n",
       "2015-07-01     NaN  551\n",
       "2015-07-02     NaN  552\n",
       "2015-07-03     NaN  553\n",
       "2015-07-04     NaN  554\n",
       "2015-07-05     NaN  555\n",
       "2015-07-06     NaN  556\n",
       "2015-07-07     NaN  557\n",
       "2015-07-08     NaN  558\n",
       "2015-07-09     NaN  559\n",
       "2015-07-10     NaN  560\n",
       "2015-07-11     NaN  561"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_data['IND'] = np.arange(model_data.shape[0])\n",
    "model_data.tail(35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model_data['IND_SQ'] = model_data['IND']**2\n",
    "x = model_data.ix[0:532, [\"IND\",\"IND_SQ\"]]\n",
    "y = model_data.ix[0:532, \"HighQ\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_test = model_data.ix[532:, [\"IND\",\"IND_SQ\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(532, 2) (532,)\n"
     ]
    }
   ],
   "source": [
    "print x.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ols = linear_model.LinearRegression(fit_intercept=True)\n",
    "ols.fit(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ols_predict = ols.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 242.22975061,  242.22165677,  242.21357583,  242.2055078 ,\n",
       "        242.19745268,  242.18941047,  242.18138116,  242.17336477,\n",
       "        242.16536128,  242.15737069,  242.14939302,  242.14142825,\n",
       "        242.13347639,  242.12553744,  242.11761139,  242.10969826,\n",
       "        242.10179803,  242.09391071,  242.08603629,  242.07817479,\n",
       "        242.07032619,  242.0624905 ,  242.05466772,  242.04685784,\n",
       "        242.03906087,  242.03127681,  242.02350566,  242.01574741,\n",
       "        242.00800208,  242.00026965])"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ols_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ -1.49671672e-02,   6.45382476e-06])"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ols.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise** Predict prices for MedQ for CA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}