notes/tfidf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example computations for TFIDF using pandas data frames\n",
    "\n",
    "Looking at the words in the next code cell, which words best distinguish the three documents?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['the', 'new', 'new', 'york', 'times'],\n",
       " ['the', 'new', 'york', 'post'],\n",
       " ['the', 'los', 'angeles', 'times']]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "\n",
    "d1 = \"the new new york times\" # (repeated new)\n",
    "d2 = \"the new york post\"\n",
    "d3 = \"the los angeles times\"\n",
    "\n",
    "docstrs = [d1,d2,d3]\n",
    "docs = [s.split() for s in docstrs]\n",
    "N = len(docs)\n",
    "docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['angeles', 'los', 'new', 'post', 'the', 'times', 'york']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniq = set(' '.join(docstrs).split())\n",
    "uniq = sorted(list(uniq))\n",
    "uniq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word vectors and term counts\n",
    "\n",
    "First, let's do this the hard way to see the actual computations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>angeles</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>los</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>post</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>times</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>york</th>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: [angeles, los, new, post, the, times, york]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf = pd.DataFrame(data=list(uniq), columns=['word'])\n",
    "tf = tf.set_index('word')\n",
    "tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_items([('the', 1), ('new', 2), ('york', 1), ('times', 1)])\n",
      "dict_items([('the', 1), ('new', 1), ('york', 1), ('post', 1)])\n",
      "dict_items([('the', 1), ('los', 1), ('angeles', 1), ('times', 1)])\n"
     ]
    }
   ],
   "source": [
    "# slow but obvious implementation\n",
    "for i,d in enumerate(docs):\n",
    "    c = Counter(d)\n",
    "    print(c.items())\n",
    "    for w,c in c.items():\n",
    "        tf.loc[w,f\"d{i+1}\"] = c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>d1</th>\n",
       "      <th>d2</th>\n",
       "      <th>d3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>angeles</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>los</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new</th>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>post</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>times</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>york</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          d1   d2   d3\n",
       "word                  \n",
       "angeles  NaN  NaN  1.0\n",
       "los      NaN  NaN  1.0\n",
       "new      2.0  1.0  NaN\n",
       "post     NaN  1.0  NaN\n",
       "the      1.0  1.0  1.0\n",
       "times    1.0  NaN  1.0\n",
       "york     1.0  1.0  NaN"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>d1</th>\n",
       "      <th>d2</th>\n",
       "      <th>d3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>angeles</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>los</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>post</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>times</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>york</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         d1  d2  d3\n",
       "word               \n",
       "angeles   0   0   1\n",
       "los       0   0   1\n",
       "new       2   1   0\n",
       "post      0   1   0\n",
       "the       1   1   1\n",
       "times     1   0   1\n",
       "york      1   1   0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf = tf.fillna(0).astype('int')\n",
    "tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>word</th>\n",
       "      <th>angeles</th>\n",
       "      <th>los</th>\n",
       "      <th>new</th>\n",
       "      <th>post</th>\n",
       "      <th>the</th>\n",
       "      <th>times</th>\n",
       "      <th>york</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>d1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "word  angeles  los  new  post  the  times  york\n",
       "d1          0    0    2     0    1      1     1\n",
       "d2          0    0    1     1    1      0     1\n",
       "d3          1    1    0     0    1      1     0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert term counts to term frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>d1</th>\n",
       "      <th>d2</th>\n",
       "      <th>d3</th>\n",
       "      <th>d1 tf</th>\n",
       "      <th>d2 tf</th>\n",
       "      <th>d3 tf</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>angeles</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>los</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>post</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>times</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>york</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         d1  d2  d3  d1 tf  d2 tf  d3 tf\n",
       "word                                    \n",
       "angeles   0   0   1    0.0   0.00   0.25\n",
       "los       0   0   1    0.0   0.00   0.25\n",
       "new       2   1   0    0.4   0.25   0.00\n",
       "post      0   1   0    0.0   0.25   0.00\n",
       "the       1   1   1    0.2   0.25   0.25\n",
       "times     1   0   1    0.2   0.00   0.25\n",
       "york      1   1   0    0.2   0.25   0.00"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf['d1 tf'] = tf['d1'] / len(docs[0])\n",
    "tf['d2 tf'] = tf['d2'] / len(docs[1])\n",
    "tf['d3 tf'] = tf['d3'] / len(docs[2])\n",
    "tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Document frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>angeles</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>los</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>post</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>times</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>york</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         doc count\n",
       "word              \n",
       "angeles          1\n",
       "los              1\n",
       "new              2\n",
       "post             1\n",
       "the              3\n",
       "times            2\n",
       "york             2"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data=uniq, columns=['word'])\n",
    "df = df.set_index('word')\n",
    "df['doc count'] = [np.sum([w in d for d in docs]) for w in uniq]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc count</th>\n",
       "      <th>df</th>\n",
       "      <th>idf</th>\n",
       "      <th>log idf</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>angeles</th>\n",
       "      <td>1</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.301030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>los</th>\n",
       "      <td>1</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.301030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new</th>\n",
       "      <td>2</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>0.124939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>post</th>\n",
       "      <td>1</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.301030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>times</th>\n",
       "      <td>2</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>0.124939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>york</th>\n",
       "      <td>2</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>0.124939</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         doc count    df       idf   log idf\n",
       "word                                        \n",
       "angeles          1  0.50  2.000000  0.301030\n",
       "los              1  0.50  2.000000  0.301030\n",
       "new              2  0.75  1.333333  0.124939\n",
       "post             1  0.50  2.000000  0.301030\n",
       "the              3  1.00  1.000000  0.000000\n",
       "times            2  0.75  1.333333  0.124939\n",
       "york             2  0.75  1.333333  0.124939"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['df'] = (df['doc count']+1) / (N+1)   # plus 1 for \"additive smoothing\"\n",
    "#df['df'] = df['doc count'] / N          # use this one for no smoothing\n",
    "df['idf'] = 1 / df['df']\n",
    "df['log idf'] = np.log10(df['idf'])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>d1 tf</th>\n",
       "      <th>d2 tf</th>\n",
       "      <th>d3 tf</th>\n",
       "      <th>df</th>\n",
       "      <th>idf</th>\n",
       "      <th>log idf</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>angeles</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.301030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>los</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.301030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new</th>\n",
       "      <td>0.4</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>0.124939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>post</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.301030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>times</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>0.124939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>york</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>0.124939</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         d1 tf  d2 tf  d3 tf    df       idf   log idf\n",
       "word                                                  \n",
       "angeles    0.0   0.00   0.25  0.50  2.000000  0.301030\n",
       "los        0.0   0.00   0.25  0.50  2.000000  0.301030\n",
       "new        0.4   0.25   0.00  0.75  1.333333  0.124939\n",
       "post       0.0   0.25   0.00  0.50  2.000000  0.301030\n",
       "the        0.2   0.25   0.25  1.00  1.000000  0.000000\n",
       "times      0.2   0.00   0.25  0.75  1.333333  0.124939\n",
       "york       0.2   0.25   0.00  0.75  1.333333  0.124939"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf = pd.concat([tf[['d1 tf','d2 tf','d3 tf']],\n",
    "                   df[['df','idf','log idf']]], axis=1)\n",
    "tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>d1 tf</th>\n",
       "      <th>d2 tf</th>\n",
       "      <th>d3 tf</th>\n",
       "      <th>df</th>\n",
       "      <th>idf</th>\n",
       "      <th>log idf</th>\n",
       "      <th>d1 tfidf</th>\n",
       "      <th>d2 tfidf</th>\n",
       "      <th>d3 tfidf</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>angeles</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.301030</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.075257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>los</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.301030</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.075257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new</th>\n",
       "      <td>0.4</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>0.124939</td>\n",
       "      <td>0.049975</td>\n",
       "      <td>0.031235</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>post</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.301030</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.075257</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.25</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>times</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>0.124939</td>\n",
       "      <td>0.024988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.031235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>york</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>0.124939</td>\n",
       "      <td>0.024988</td>\n",
       "      <td>0.031235</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         d1 tf  d2 tf  d3 tf    df       idf   log idf  d1 tfidf  d2 tfidf  \\\n",
       "word                                                                         \n",
       "angeles    0.0   0.00   0.25  0.50  2.000000  0.301030  0.000000  0.000000   \n",
       "los        0.0   0.00   0.25  0.50  2.000000  0.301030  0.000000  0.000000   \n",
       "new        0.4   0.25   0.00  0.75  1.333333  0.124939  0.049975  0.031235   \n",
       "post       0.0   0.25   0.00  0.50  2.000000  0.301030  0.000000  0.075257   \n",
       "the        0.2   0.25   0.25  1.00  1.000000  0.000000  0.000000  0.000000   \n",
       "times      0.2   0.00   0.25  0.75  1.333333  0.124939  0.024988  0.000000   \n",
       "york       0.2   0.25   0.00  0.75  1.333333  0.124939  0.024988  0.031235   \n",
       "\n",
       "         d3 tfidf  \n",
       "word               \n",
       "angeles  0.075257  \n",
       "los      0.075257  \n",
       "new      0.000000  \n",
       "post     0.000000  \n",
       "the      0.000000  \n",
       "times    0.031235  \n",
       "york     0.000000  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf['d1 tfidf'] = tfidf['d1 tf'] * tfidf['log idf']\n",
    "tfidf['d2 tfidf'] = tfidf['d2 tf'] * tfidf['log idf']\n",
    "tfidf['d3 tfidf'] = tfidf['d3 tf'] * tfidf['log idf']\n",
    "tfidf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CountVectorizer\n",
    "\n",
    "Ok, now get term counts the the easy way."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['the new new york times', 'the new york post', 'the los angeles times']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docstrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x7 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 12 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "vectorizer = CountVectorizer()\n",
    "\n",
    "X = vectorizer.fit_transform(docstrs)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 4)\t1\n",
      "  (0, 2)\t2\n",
      "  (0, 6)\t1\n",
      "  (0, 5)\t1\n",
      "  (1, 4)\t1\n",
      "  (1, 2)\t1\n",
      "  (1, 6)\t1\n",
      "  (1, 3)\t1\n",
      "  (2, 4)\t1\n",
      "  (2, 5)\t1\n",
      "  (2, 1)\t1\n",
      "  (2, 0)\t1\n"
     ]
    }
   ],
   "source": [
    "print(X) # maps (i,j) key to value as sparse matrix implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 2, 0, 1, 1, 1],\n",
       "       [0, 0, 1, 1, 1, 0, 1],\n",
       "       [1, 1, 0, 0, 1, 1, 0]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['angeles', 'los', 'new', 'post', 'the', 'times', 'york']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>angeles</th>\n",
       "      <th>los</th>\n",
       "      <th>new</th>\n",
       "      <th>post</th>\n",
       "      <th>the</th>\n",
       "      <th>times</th>\n",
       "      <th>york</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   angeles  los  new  post  the  times  york\n",
       "0        0    0    2     0    1      1     1\n",
       "1        0    0    1     1    1      0     1\n",
       "2        1    1    0     0    1      1     0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(data=X.toarray(), columns=vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare to manually computed values. Heh, we match!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>word</th>\n",
       "      <th>angeles</th>\n",
       "      <th>los</th>\n",
       "      <th>new</th>\n",
       "      <th>post</th>\n",
       "      <th>the</th>\n",
       "      <th>times</th>\n",
       "      <th>york</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>d1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "word  angeles  los  new  post  the  times  york\n",
       "d1          0    0    2     0    1      1     1\n",
       "d2          0    0    1     1    1      0     1\n",
       "d3          1    1    0     0    1      1     0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf[['d1','d2','d3']].T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TfidfTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>angeles</th>\n",
       "      <th>los</th>\n",
       "      <th>new</th>\n",
       "      <th>post</th>\n",
       "      <th>the</th>\n",
       "      <th>times</th>\n",
       "      <th>york</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.778317</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.302216</td>\n",
       "      <td>0.389158</td>\n",
       "      <td>0.389158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.480458</td>\n",
       "      <td>0.631745</td>\n",
       "      <td>0.373119</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.480458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.584483</td>\n",
       "      <td>0.584483</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.345205</td>\n",
       "      <td>0.444514</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    angeles       los       new      post       the     times      york\n",
       "0  0.000000  0.000000  0.778317  0.000000  0.302216  0.389158  0.389158\n",
       "1  0.000000  0.000000  0.480458  0.631745  0.373119  0.000000  0.480458\n",
       "2  0.584483  0.584483  0.000000  0.000000  0.345205  0.444514  0.000000"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "trans = TfidfTransformer()\n",
    "D = trans.fit_transform(X).toarray()  # X is word count matrix from above\n",
    "pd.DataFrame(data=D, columns=vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare to manually computed. Note that sklearn says \"*Note that the idf formula above differs from the standard textbook notation that defines the idf as idf(t) = log [ n / (df(t) + 1) ]*\" so we're a bit different."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>word</th>\n",
       "      <th>angeles</th>\n",
       "      <th>los</th>\n",
       "      <th>new</th>\n",
       "      <th>post</th>\n",
       "      <th>the</th>\n",
       "      <th>times</th>\n",
       "      <th>york</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>d1 tfidf</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.049975</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.024988</td>\n",
       "      <td>0.024988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d2 tfidf</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.031235</td>\n",
       "      <td>0.075257</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.031235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d3 tfidf</th>\n",
       "      <td>0.075257</td>\n",
       "      <td>0.075257</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.031235</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "word       angeles       los       new      post  the     times      york\n",
       "d1 tfidf  0.000000  0.000000  0.049975  0.000000  0.0  0.024988  0.024988\n",
       "d2 tfidf  0.000000  0.000000  0.031235  0.075257  0.0  0.000000  0.031235\n",
       "d3 tfidf  0.075257  0.075257  0.000000  0.000000  0.0  0.031235  0.000000"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf[['d1 tfidf', 'd2 tfidf', 'd3 tfidf']].T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What can we conclude?\n",
    "\n",
    "For these three documents, the words with the highest TFIDF scores are *new*, *post*, and *angeles* (or *los*). That corresponds to our intuition that those words distinguish between the documents best.  Words like *the* appear in every document so they are not good distinguishing features.  Because *york* appears in two of three documents, it's score was attenuated but not zero doubt."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {},
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "threshold": 4,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}