POS_Tagging.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# POS_TAGGER\n",
    "## - step 1: Read and preprocess the dataset\n",
    "## - step 2: Build a Most Frequent Class (MFC) tagger to use as a       baseline\n",
    "## - step 3: Build an HMM POS Tagger\n",
    "## - step 4: RNN, LSTM (optional)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "import numpy as np \n",
    "from itertools import chain\n",
    "from collections import Counter, defaultdict\n",
    "from pomegranate import State, HiddenMarkovModel, DiscreteDistribution\n",
    "import pandas as pd\n",
    "import sklearn as skl \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import string\n",
    "import math\n",
    "from collections import defaultdict"
   ]
  },
  {
   "source": [
    "## Working with data file and create vocabulary"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "### Read data file"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(\"WSJ_02-21.pos\", 'r') as f:\n",
    "    lines = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "In\tIN\n\nan\tDT\n\nOct.\tNNP\n\n19\tCD\n\nreview\tNN\n\nof\tIN\n\n``\t``\n\nThe\tDT\n\nMisanthrope\tNN\n\n''\t''\n\nat\tIN\n\nChicago\tNNP\n\n's\tPOS\n\nGoodman\tNNP\n\nTheatre\tNNP\n\n(\t(\n\n``\t``\n\nRevitalized\tVBN\n\nClassics\tNNS\n\nTake\tVBP\n\n"
     ]
    }
   ],
   "source": [
    "for i in range(20):\n",
    "    print(lines[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = [line.split('\\t')[0] for line in lines]\n",
    "# words"
   ]
  },
  {
   "source": [
    "### Create vocabulary file"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq = defaultdict(int)\n",
    "\n",
    "for word in words:\n",
    "    freq[word] +=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "1870"
      ]
     },
     "metadata": {},
     "execution_count": 165
    }
   ],
   "source": [
    "freq['more']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocabulary = [k for k, v in freq.items() if (v > 1 and k != '\\n')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocabulary.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vocabulary_file = open('vocabulary_file.txt', \"w+\")\n",
    "for i in vocabulary:\n",
    "    vocabulary_file = open('vocabulary_file.txt', \"a+\")\n",
    "    vocabulary_file.writelines(i + '\\n')\n",
    "# for i in range(0,10):\n",
    "#     print(vocabulary[i])"
   ]
  },
  {
   "source": [
    "### Processing new text sources"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "#### working words don't exist in the vocabulary"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "def assign_unknow(word):\n",
    "\n",
    "    #punctuation characters\n",
    "    punct = set(string.punctuation)\n",
    "\n",
    "    #suffixes\n",
    "    noun_suffix = [\"action\", \"age\", \"ance\", \"cy\", \"dom\", \"ee\", \"ence\", \"er\", \"hood\", \"ion\", \"ism\", \"ist\", \"ity\", \"ling\", \"ment\", \"ness\", \"or\", \"ry\", \"scape\", \"ship\", \"ty\"]\n",
    "    verb_suffix = [\"ate\", \"ify\", \"ise\", \"ize\"]\n",
    "    adj_suffix = [\"able\", \"ese\", \"ful\", \"i\", \"ian\", \"ible\", \"ic\", \"ish\", \"ive\", \"less\", \"ly\", \"ous\"]\n",
    "    adv_suffix = [\"ward\", \"wards\", \"wise\"]\n",
    "    \n",
    "    if any(char.isdigit() for char in word):\n",
    "        return \"--unknow_digit--\"\n",
    "\n",
    "    elif any(char in punct for char in word):\n",
    "        return \"--unknow_punct--\"\n",
    "\n",
    "    elif any(char.isupper() for char in word):\n",
    "        return \"--unknow_upper--\"\n",
    "\n",
    "    elif any(word.endswith(suffix) for suffix in noun_suffix):\n",
    "        return \"--unknow_noun--\" \n",
    "    \n",
    "    elif any(word.endswith(suffix) for suffix in verb_suffix):\n",
    "        return \"--unknow_verb--\"\n",
    "\n",
    "    elif any(word.endswith(suffix) for suffix in adj_suffix):\n",
    "        return \"--unknow_adj--\"\n",
    "    \n",
    "    elif any(word.endswith(suffix) for suffix in adv_suffix):\n",
    "        return \"--unknow_adv--\"\n",
    "\n",
    "    return \"--unknow--\""
   ]
  },
  {
   "source": [
    "### Getting the correct tag for a word"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_tag(line, vocabulary):\n",
    "    if not line.split():\n",
    "        word = \"--n--\"\n",
    "        tag = \"--s--\"\n",
    "    else:\n",
    "        word, tag = line.split()\n",
    "        if word not in vocabulary:\n",
    "            word = assign_unknow(word)\n",
    "\n",
    "    return word, tag "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "('--n--', '--s--')\n('In', 'IN')\n('--unknow--', 'NN')\n('--unknow_verb--', 'VB')\n"
     ]
    }
   ],
   "source": [
    "print(get_word_tag('\\n', vocabulary))\n",
    "print(get_word_tag('In\\tIN\\n', vocabulary))\n",
    "print(get_word_tag('tardigrade\\tNN\\n', vocabulary))\n",
    "print(get_word_tag('scrutinize\\tVB\\n', vocabulary))"
   ]
  },
  {
   "source": [
    "## Working with tags and Numpy"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd"
   ]
  },
  {
   "source": [
    "### test with only 3 tags (RB, NN, TO)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = ['RB', 'NN', 'TO']"
   ]
  },
  {
   "source": [
    "### testing with transition_counts, which counts the number of times a particular tag happend next to another. The keys of dictionary have the form (previous_tag, tag) and the values are the frequency of occurrences. The trainsition_dictionary just works with tags only"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for example: define a transition_counts dictionary with random value:\n",
    "transition_counts = {\n",
    "    ('NN', 'NN'): 16241,\n",
    "    ('RB', 'RB'): 2263,\n",
    "    ('TO', 'TO'): 2,\n",
    "    ('NN', 'TO'): 5256,\n",
    "    ('RB', 'TO'): 855,\n",
    "    ('TO', 'NN'): 734,\n",
    "    ('NN', 'RB'): 2431,\n",
    "    ('RB', 'NN'): 358,\n",
    "    ('TO', 'RB'): 200\n",
    "}\n"
   ]
  },
  {
   "source": [
    "### Using numpy of matrix creation:"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "3\n[[0. 0. 0.]\n [0. 0. 0.]\n [0. 0. 0.]]\n(3, 3)\n"
     ]
    }
   ],
   "source": [
    "num_tags = len(tags)\n",
    "print(num_tags)\n",
    "\n",
    "transition_matrix = np.zeros((num_tags, num_tags))\n",
    "\n",
    "print(transition_matrix)\n",
    "print(transition_matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['NN', 'RB', 'TO']"
      ]
     },
     "metadata": {},
     "execution_count": 176
    }
   ],
   "source": [
    "sorted_tags = sorted(tags)\n",
    "\n",
    "sorted_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[1.6241e+04, 2.4310e+03, 5.2560e+03],\n",
       "       [3.5800e+02, 2.2630e+03, 8.5500e+02],\n",
       "       [7.3400e+02, 2.0000e+02, 2.0000e+00]])"
      ]
     },
     "metadata": {},
     "execution_count": 177
    }
   ],
   "source": [
    "for i in range(num_tags):\n",
    "    for j in range(num_tags):\n",
    "        tag_tuple = (sorted_tags[i], sorted_tags[j])\n",
    "\n",
    "        transition_matrix[i, j] = transition_counts.get(tag_tuple)\n",
    "\n",
    "transition_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "         NN      RB      TO\nNN  16241.0  2431.0  5256.0\nRB    358.0  2263.0   855.0\nTO    734.0   200.0     2.0\n"
     ]
    }
   ],
   "source": [
    "def print_transition_matrix(matrix):\n",
    "    print(pd.DataFrame(matrix, index = sorted_tags, columns = sorted_tags))\n",
    "\n",
    "print_transition_matrix(transition_matrix)"
   ]
  },
  {
   "source": [
    "### Working with Numpy for matrix manipulation"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "rows_sum = transition_matrix.sum(axis = 1, keepdims = True)\n",
    "\n",
    "rows_sum"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 179,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[23928.],\n",
       "       [ 3476.],\n",
       "       [  936.]])"
      ]
     },
     "metadata": {},
     "execution_count": 179
    }
   ]
  },
  {
   "source": [
    "# normalize the matrix \n",
    "rows_sum = transition_matrix.sum(axis = 1, keepdims = True)\n",
    "\n",
    "transition_matrix = transition_matrix / rows_sum\n",
    "\n",
    "print_transition_matrix(transition_matrix)"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 180,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "          NN        RB        TO\nNN  0.678745  0.101596  0.219659\nRB  0.102992  0.651036  0.245972\nTO  0.784188  0.213675  0.002137\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Create rules use for unknown word\n",
    "\n",
    "noun_suffix = [\"action\", \"age\", \"ance\", \"cy\", \"dom\", \"ee\", \"ence\", \"er\", \"hood\", \"ion\", \"ism\", \"ist\", \"ity\", \"ling\", \"ment\", \"ness\", \"or\", \"ry\", \"scape\", \"ship\", \"ty\"]\n",
    "verb_suffix = [\"ate\", \"ify\", \"ise\", \"ize\"]\n",
    "adj_suffix = [\"able\", \"ese\", \"ful\", \"i\", \"ian\", \"ible\", \"ic\", \"ish\", \"ive\", \"less\", \"ly\", \"ous\"]\n",
    "adv_suffix = [\"ward\", \"wards\", \"wise\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "punct = set(string.punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "def assign_unknow(word):\n",
    "\n",
    "    noun_suffix = [\"action\", \"age\", \"ance\", \"cy\", \"dom\", \"ee\", \"ence\", \"er\", \"hood\", \"ion\", \"ism\", \"ist\", \"ity\", \"ling\", \"ment\", \"ness\", \"or\", \"ry\", \"scape\", \"ship\", \"ty\"]\n",
    "    verb_suffix = [\"ate\", \"ify\", \"ise\", \"ize\"]\n",
    "    adj_suffix = [\"able\", \"ese\", \"ful\", \"i\", \"ian\", \"ible\", \"ic\", \"ish\", \"ive\", \"less\", \"ly\", \"ous\"]\n",
    "    adv_suffix = [\"ward\", \"wards\", \"wise\"]\n",
    "\n",
    "    if any(char.isdigit() for char in word):\n",
    "        return \"--unknow_digit--\"\n",
    "\n",
    "    elif any(char in punct for char in word):\n",
    "        return \"--unknow_punct--\"\n",
    "\n",
    "    elif any(char.isupper() for char in word):\n",
    "        return \"--unknow_upper--\"\n",
    "\n",
    "    elif any(word.endswith(suffix) for suffix in noun_suffix):\n",
    "        return \"--unknow_noun--\"\n",
    "\n",
    "    elif any(word.endswith(suffix) for suffix in verb_suffix):\n",
    "        return \"--unknow_verb--\"\n",
    "\n",
    "    elif any(word.endswith(suffix) for suffix in adj_suffix):\n",
    "        return \"--unknow_adj--\"\n",
    "\n",
    "    elif any(word.endswith(suffix) for suffix in adv_suffix):\n",
    "        return \"--unknow_adv--\"\n",
    "\n",
    "    return \"--unknow--\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_tag(line, vocab):\n",
    "    if not line.split():\n",
    "        word = \"--n--\"\n",
    "        tag = \"--s--\"\n",
    "    else:\n",
    "        word, tag = line.split()\n",
    "        if word not in vocab:\n",
    "            word = assign_unknow(word)\n",
    "        return word, tag\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "None\n('In', 'IN')\n('--unknow--', 'NN')\n('--unknow_verb--', 'VB')\n"
     ]
    }
   ],
   "source": [
    "with open(\"hmm_vocab.txt\", 'r') as f:\n",
    "    vocabulary = f.read().split('\\n')\n",
    "\n",
    "\n",
    "print(get_word_tag('\\n', vocabulary))\n",
    "print(get_word_tag('In\\tIN\\n', vocabulary))\n",
    "print(get_word_tag('tardigrade\\tNN\\n', vocabulary))\n",
    "print(get_word_tag('scrutinize\\tVB\\n', vocabulary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# print(vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(\"WSJ_02-21.pos\", 'r') as f:\n",
    "    training_set = f.readlines()\n",
    "\n",
    "# print(training_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(vocab, data_fp):\n",
    "    \"\"\"\n",
    "    Preprocess data\n",
    "    \"\"\"\n",
    "    orig = []\n",
    "    prep = []\n",
    "\n",
    "    # Read data\n",
    "    with open(data_fp, \"r\") as data_file:\n",
    "\n",
    "        for cnt, word in enumerate(data_file):\n",
    "\n",
    "            # End of sentence\n",
    "            if not word.split():\n",
    "                orig.append(word.strip())\n",
    "                word = \"--n--\"\n",
    "                prep.append(word)\n",
    "                continue\n",
    "\n",
    "            # Handle unknown words\n",
    "            elif word.strip() not in vocab:\n",
    "                orig.append(word.strip())\n",
    "                word = assign_unknow(word)\n",
    "                prep.append(word)\n",
    "                continue\n",
    "\n",
    "            else:\n",
    "                orig.append(word.strip())\n",
    "                prep.append(word.strip())\n",
    "\n",
    "    assert(len(orig) == len(open(data_fp, \"r\").readlines()))\n",
    "    assert(len(prep) == len(open(data_fp, \"r\").readlines()))\n",
    "\n",
    "    return orig, prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'': 0,\n",
       " '!': 1,\n",
       " '#': 2,\n",
       " '$': 3,\n",
       " '%': 4,\n",
       " '&': 5,\n",
       " \"'\": 6,\n",
       " \"''\": 7,\n",
       " \"'40s\": 8,\n",
       " \"'60s\": 9,\n",
       " \"'70s\": 10,\n",
       " \"'80s\": 11,\n",
       " \"'86\": 12,\n",
       " \"'90s\": 13,\n",
       " \"'N\": 14,\n",
       " \"'S\": 15,\n",
       " \"'d\": 16,\n",
       " \"'em\": 17,\n",
       " \"'ll\": 18,\n",
       " \"'m\": 19,\n",
       " \"'n'\": 20,\n",
       " \"'re\": 21,\n",
       " \"'s\": 22,\n",
       " \"'til\": 23,\n",
       " \"'ve\": 24,\n",
       " '(': 25,\n",
       " ')': 26,\n",
       " ',': 27,\n",
       " '-': 28,\n",
       " '--': 29,\n",
       " '--n--': 30,\n",
       " '--unknow--': 31,\n",
       " '--unknow_adj--': 32,\n",
       " '--unknow_adv--': 33,\n",
       " '--unknow_digit--': 34,\n",
       " '--unknow_noun--': 35,\n",
       " '--unknow_punct--': 36,\n",
       " '--unknow_upper--': 37,\n",
       " '--unknow_verb--': 38,\n",
       " '.': 39,\n",
       " '...': 40,\n",
       " '0.01': 41,\n",
       " '0.0108': 42,\n",
       " '0.02': 43,\n",
       " '0.03': 44,\n",
       " '0.05': 45,\n",
       " '0.1': 46,\n",
       " '0.10': 47,\n",
       " '0.12': 48,\n",
       " '0.13': 49,\n",
       " '0.15': 50,\n",
       " '0.17': 51,\n",
       " '0.19': 52,\n",
       " '0.2': 53,\n",
       " '0.24': 54,\n",
       " '0.25': 55,\n",
       " '0.3': 56,\n",
       " '0.32': 57,\n",
       " '0.375': 58,\n",
       " '0.4': 59,\n",
       " '0.43': 60,\n",
       " '0.45': 61,\n",
       " '0.5': 62,\n",
       " '0.53': 63,\n",
       " '0.59': 64,\n",
       " '0.6': 65,\n",
       " '0.60': 66,\n",
       " '0.7': 67,\n",
       " '0.71': 68,\n",
       " '0.75': 69,\n",
       " '0.8': 70,\n",
       " '0.88': 71,\n",
       " '0.9': 72,\n",
       " '0.94': 73,\n",
       " '0.95': 74,\n",
       " '1': 75,\n",
       " '1,000': 76,\n",
       " '1,012': 77,\n",
       " '1,015': 78,\n",
       " '1,040': 79,\n",
       " '1,050': 80,\n",
       " '1,100': 81,\n",
       " '1,111': 82,\n",
       " '1,150,000': 83,\n",
       " '1,200': 84,\n",
       " '1,250': 85,\n",
       " '1,250,000': 86,\n",
       " '1,300': 87,\n",
       " '1,365,226': 88,\n",
       " '1,400': 89,\n",
       " '1,500': 90,\n",
       " '1,600': 91,\n",
       " '1,700': 92,\n",
       " '1,750': 93,\n",
       " '1,800': 94,\n",
       " '1,828,000': 95,\n",
       " '1,850': 96,\n",
       " '1,859': 97,\n",
       " '1,900': 98,\n",
       " '1-2-3': 99,\n",
       " '1-for-10': 100,\n",
       " '1-to-1': 101,\n",
       " '1.0': 102,\n",
       " '1.01': 103,\n",
       " '1.02': 104,\n",
       " '1.03': 105,\n",
       " '1.04': 106,\n",
       " '1.05': 107,\n",
       " '1.06': 108,\n",
       " '1.07': 109,\n",
       " '1.08': 110,\n",
       " '1.09': 111,\n",
       " '1.1': 112,\n",
       " '1.10': 113,\n",
       " '1.11': 114,\n",
       " '1.12': 115,\n",
       " '1.125': 116,\n",
       " '1.13': 117,\n",
       " '1.14': 118,\n",
       " '1.15': 119,\n",
       " '1.16': 120,\n",
       " '1.17': 121,\n",
       " '1.18': 122,\n",
       " '1.19': 123,\n",
       " '1.2': 124,\n",
       " '1.20': 125,\n",
       " '1.21': 126,\n",
       " '1.22': 127,\n",
       " '1.23': 128,\n",
       " '1.24': 129,\n",
       " '1.25': 130,\n",
       " '1.26': 131,\n",
       " '1.27': 132,\n",
       " '1.28': 133,\n",
       " '1.29': 134,\n",
       " '1.3': 135,\n",
       " '1.30': 136,\n",
       " '1.31': 137,\n",
       " '1.32': 138,\n",
       " '1.34': 139,\n",
       " '1.35': 140,\n",
       " '1.36': 141,\n",
       " '1.37': 142,\n",
       " '1.375': 143,\n",
       " '1.38': 144,\n",
       " '1.39': 145,\n",
       " '1.4': 146,\n",
       " '1.40': 147,\n",
       " '1.41': 148,\n",
       " '1.42': 149,\n",
       " '1.43': 150,\n",
       " '1.44': 151,\n",
       " '1.45': 152,\n",
       " '1.46': 153,\n",
       " '1.47': 154,\n",
       " '1.48': 155,\n",
       " '1.49': 156,\n",
       " '1.5': 157,\n",
       " '1.50': 158,\n",
       " '1.51': 159,\n",
       " '1.52': 160,\n",
       " '1.53': 161,\n",
       " '1.54': 162,\n",
       " '1.55': 163,\n",
       " '1.56': 164,\n",
       " '1.5753': 165,\n",
       " '1.5765': 166,\n",
       " '1.5795': 167,\n",
       " '1.58': 168,\n",
       " '1.5820': 169,\n",
       " '1.5825': 170,\n",
       " '1.59': 171,\n",
       " '1.5920': 172,\n",
       " '1.6': 173,\n",
       " '1.60': 174,\n",
       " '1.6030': 175,\n",
       " '1.6055': 176,\n",
       " '1.6145': 177,\n",
       " '1.62': 178,\n",
       " '1.625': 179,\n",
       " '1.63': 180,\n",
       " '1.64': 181,\n",
       " '1.65': 182,\n",
       " '1.66': 183,\n",
       " '1.67': 184,\n",
       " '1.69': 185,\n",
       " '1.7': 186,\n",
       " '1.70': 187,\n",
       " '1.71': 188,\n",
       " '1.72': 189,\n",
       " '1.73': 190,\n",
       " '1.74': 191,\n",
       " '1.75': 192,\n",
       " '1.76': 193,\n",
       " '1.77': 194,\n",
       " '1.78': 195,\n",
       " '1.79': 196,\n",
       " '1.8': 197,\n",
       " '1.80': 198,\n",
       " '1.81': 199,\n",
       " '1.82': 200,\n",
       " '1.8200': 201,\n",
       " '1.83': 202,\n",
       " '1.8300': 203,\n",
       " '1.8340': 204,\n",
       " '1.8353': 205,\n",
       " '1.8355': 206,\n",
       " '1.84': 207,\n",
       " '1.8400': 208,\n",
       " '1.8415': 209,\n",
       " '1.8470': 210,\n",
       " '1.8485': 211,\n",
       " '1.85': 212,\n",
       " '1.8578': 213,\n",
       " '1.86': 214,\n",
       " '1.8667': 215,\n",
       " '1.8685': 216,\n",
       " '1.87': 217,\n",
       " '1.875': 218,\n",
       " '1.88': 219,\n",
       " '1.89': 220,\n",
       " '1.9': 221,\n",
       " '1.90': 222,\n",
       " '1.91': 223,\n",
       " '1.92': 224,\n",
       " '1.93': 225,\n",
       " '1.94': 226,\n",
       " '1.95': 227,\n",
       " '10': 228,\n",
       " '10,000': 229,\n",
       " '10-11': 230,\n",
       " '10-a-share': 231,\n",
       " '10-cent-a-share': 232,\n",
       " '10-month': 233,\n",
       " '10-year': 234,\n",
       " '10-year-old': 235,\n",
       " '10.03': 236,\n",
       " '10.05': 237,\n",
       " '10.1': 238,\n",
       " '10.14': 239,\n",
       " '10.2': 240,\n",
       " '10.3': 241,\n",
       " '10.35': 242,\n",
       " '10.37': 243,\n",
       " '10.4': 244,\n",
       " '10.48': 245,\n",
       " '10.5': 246,\n",
       " '10.59': 247,\n",
       " '10.6': 248,\n",
       " '10.625': 249,\n",
       " '10.7': 250,\n",
       " '10.77': 251,\n",
       " '10.8': 252,\n",
       " '10.9': 253,\n",
       " '100': 254,\n",
       " '100,000': 255,\n",
       " '100-Share': 256,\n",
       " '100-share': 257,\n",
       " '100-stock': 258,\n",
       " '100.2': 259,\n",
       " '100.4': 260,\n",
       " '101': 261,\n",
       " '101.4': 262,\n",
       " '102': 263,\n",
       " '102.1': 264,\n",
       " '102.625': 265,\n",
       " '103': 266,\n",
       " '103,000': 267,\n",
       " '104': 268,\n",
       " '105': 269,\n",
       " '105.4': 270,\n",
       " '106': 271,\n",
       " '107': 272,\n",
       " '108': 273,\n",
       " '108.4': 274,\n",
       " '109': 275,\n",
       " '109.85': 276,\n",
       " '10:30': 277,\n",
       " '10:40': 278,\n",
       " '10\\\\/32': 279,\n",
       " '10th': 280,\n",
       " '11': 281,\n",
       " '11,000': 282,\n",
       " '11.04': 283,\n",
       " '11.1': 284,\n",
       " '11.2': 285,\n",
       " '11.25': 286,\n",
       " '11.3': 287,\n",
       " '11.38': 288,\n",
       " '11.4': 289,\n",
       " '11.5': 290,\n",
       " '11.53': 291,\n",
       " '11.6': 292,\n",
       " '11.60': 293,\n",
       " '11.625': 294,\n",
       " '11.7': 295,\n",
       " '11.8': 296,\n",
       " '11.9': 297,\n",
       " '11.95': 298,\n",
       " '110': 299,\n",
       " '110,000': 300,\n",
       " '110.6': 301,\n",
       " '111': 302,\n",
       " '111.48': 303,\n",
       " '112': 304,\n",
       " '112.5': 305,\n",
       " '113': 306,\n",
       " '114': 307,\n",
       " '114.3': 308,\n",
       " '114.4': 309,\n",
       " '115': 310,\n",
       " '116': 311,\n",
       " '117': 312,\n",
       " '117.3': 313,\n",
       " '118': 314,\n",
       " '119': 315,\n",
       " '119.88': 316,\n",
       " '11\\\\/16': 317,\n",
       " '11\\\\/32': 318,\n",
       " '11th': 319,\n",
       " '12': 320,\n",
       " '12,000': 321,\n",
       " '12,500': 322,\n",
       " '12-month': 323,\n",
       " '12-year': 324,\n",
       " '12-year-old': 325,\n",
       " '12.2': 326,\n",
       " '12.3': 327,\n",
       " '12.4': 328,\n",
       " '12.45': 329,\n",
       " '12.5': 330,\n",
       " '12.6': 331,\n",
       " '12.7': 332,\n",
       " '12.75': 333,\n",
       " '12.8': 334,\n",
       " '12.9': 335,\n",
       " '12.95': 336,\n",
       " '120': 337,\n",
       " '120,000': 338,\n",
       " '120-day': 339,\n",
       " '120.7': 340,\n",
       " '121': 341,\n",
       " '122': 342,\n",
       " '122.7': 343,\n",
       " '123': 344,\n",
       " '123.5': 345,\n",
       " '1230.80': 346,\n",
       " '124': 347,\n",
       " '124,875': 348,\n",
       " '1247.87': 349,\n",
       " '125': 350,\n",
       " '125,000': 351,\n",
       " '1254.27': 352,\n",
       " '127': 353,\n",
       " '127.5': 354,\n",
       " '128': 355,\n",
       " '129.49': 356,\n",
       " '12:01': 357,\n",
       " '12\\\\/32': 358,\n",
       " '13': 359,\n",
       " '13,000': 360,\n",
       " '13,120': 361,\n",
       " '13-week': 362,\n",
       " '13.05': 363,\n",
       " '13.1': 364,\n",
       " '13.2': 365,\n",
       " '13.32': 366,\n",
       " '13.35': 367,\n",
       " '13.4': 368,\n",
       " '13.5': 369,\n",
       " '13.50': 370,\n",
       " '13.6': 371,\n",
       " '13.625': 372,\n",
       " '13.7': 373,\n",
       " '13.71': 374,\n",
       " '13.75': 375,\n",
       " '13.8': 376,\n",
       " '13.94': 377,\n",
       " '130': 378,\n",
       " '130,000': 379,\n",
       " '131': 380,\n",
       " '132': 381,\n",
       " '132.8': 382,\n",
       " '133': 383,\n",
       " '134': 384,\n",
       " '134.8': 385,\n",
       " '135': 386,\n",
       " '136': 387,\n",
       " '136.4': 388,\n",
       " '137': 389,\n",
       " '137.6': 390,\n",
       " '138': 391,\n",
       " '139': 392,\n",
       " '13\\\\/16': 393,\n",
       " '13th': 394,\n",
       " '14': 395,\n",
       " '14,000': 396,\n",
       " '14-year-old': 397,\n",
       " '14.06': 398,\n",
       " '14.1': 399,\n",
       " '14.2': 400,\n",
       " '14.25': 401,\n",
       " '14.3': 402,\n",
       " '14.5': 403,\n",
       " '14.6': 404,\n",
       " '14.7': 405,\n",
       " '14.75': 406,\n",
       " '14.8': 407,\n",
       " '140': 408,\n",
       " '140,000': 409,\n",
       " '141': 410,\n",
       " '141.45': 411,\n",
       " '141.52': 412,\n",
       " '141.55': 413,\n",
       " '141.65': 414,\n",
       " '141.70': 415,\n",
       " '141.80': 416,\n",
       " '141.90': 417,\n",
       " '142': 418,\n",
       " '142.10': 419,\n",
       " '142.43': 420,\n",
       " '142.70': 421,\n",
       " '142.75': 422,\n",
       " '142.85': 423,\n",
       " '143': 424,\n",
       " '144': 425,\n",
       " '145': 426,\n",
       " '146': 427,\n",
       " '146.8': 428,\n",
       " '1466.29': 429,\n",
       " '148': 430,\n",
       " '149': 431,\n",
       " '14th': 432,\n",
       " '15': 433,\n",
       " '15,000': 434,\n",
       " '15-a-share': 435,\n",
       " '15-year': 436,\n",
       " '15.06': 437,\n",
       " '15.1': 438,\n",
       " '15.125': 439,\n",
       " '15.2': 440,\n",
       " '15.25': 441,\n",
       " '15.3': 442,\n",
       " '15.375': 443,\n",
       " '15.5': 444,\n",
       " '15.50': 445,\n",
       " '15.6': 446,\n",
       " '15.625': 447,\n",
       " '15.7': 448,\n",
       " '15.72': 449,\n",
       " '15.75': 450,\n",
       " '15.80': 451,\n",
       " '15.82': 452,\n",
       " '15.9': 453,\n",
       " '15.97': 454,\n",
       " '150': 455,\n",
       " '150,000': 456,\n",
       " '150-member': 457,\n",
       " '150.3': 458,\n",
       " '151': 459,\n",
       " '151,000': 460,\n",
       " '151.20': 461,\n",
       " '153': 462,\n",
       " '154': 463,\n",
       " '154.2': 464,\n",
       " '155': 465,\n",
       " '155,650,000': 466,\n",
       " '156': 467,\n",
       " '156.7': 468,\n",
       " '157': 469,\n",
       " '158': 470,\n",
       " '15\\\\/16': 471,\n",
       " '15\\\\/32': 472,\n",
       " '16': 473,\n",
       " '16,000': 474,\n",
       " '16-bit': 475,\n",
       " '16.1': 476,\n",
       " '16.2': 477,\n",
       " '16.3': 478,\n",
       " '16.375': 479,\n",
       " '16.4': 480,\n",
       " '16.40': 481,\n",
       " '16.5': 482,\n",
       " '16.6': 483,\n",
       " '16.75': 484,\n",
       " '16.9': 485,\n",
       " '16.95': 486,\n",
       " '160': 487,\n",
       " '160,000': 488,\n",
       " '161': 489,\n",
       " '161.1': 490,\n",
       " '161.5': 491,\n",
       " '162': 492,\n",
       " '162,000': 493,\n",
       " '163': 494,\n",
       " '163-member': 495,\n",
       " '164,830,000': 496,\n",
       " '165': 497,\n",
       " '166': 498,\n",
       " '166,900,000': 499,\n",
       " '166.9': 500,\n",
       " '167': 501,\n",
       " '168': 502,\n",
       " '16\\\\/32': 503,\n",
       " '16th': 504,\n",
       " '17': 505,\n",
       " '17,000': 506,\n",
       " '17-store': 507,\n",
       " '17.01': 508,\n",
       " '17.1': 509,\n",
       " '17.2': 510,\n",
       " '17.3': 511,\n",
       " '17.4': 512,\n",
       " '17.5': 513,\n",
       " '17.50': 514,\n",
       " '17.6': 515,\n",
       " '17.8': 516,\n",
       " '17.9': 517,\n",
       " '17.95': 518,\n",
       " '170': 519,\n",
       " '170,330,000': 520,\n",
       " '170.4': 521,\n",
       " '171': 522,\n",
       " '172': 523,\n",
       " '172.2': 524,\n",
       " '172.5': 525,\n",
       " '173.1': 526,\n",
       " '174': 527,\n",
       " '175': 528,\n",
       " '175,000': 529,\n",
       " '176': 530,\n",
       " '176,100,000': 531,\n",
       " '177': 532,\n",
       " '177.5': 533,\n",
       " '178': 534,\n",
       " '178.375': 535,\n",
       " '178.5': 536,\n",
       " '178.9': 537,\n",
       " '179': 538,\n",
       " '17\\\\/32': 539,\n",
       " '17th-century': 540,\n",
       " '18': 541,\n",
       " '18,000': 542,\n",
       " '18.1': 543,\n",
       " '18.375': 544,\n",
       " '18.4': 545,\n",
       " '18.5': 546,\n",
       " '18.50': 547,\n",
       " '18.65': 548,\n",
       " '18.7': 549,\n",
       " '18.75': 550,\n",
       " '18.9': 551,\n",
       " '18.95': 552,\n",
       " '180': 553,\n",
       " '180,000': 554,\n",
       " '181': 555,\n",
       " '182': 556,\n",
       " '182-day': 557,\n",
       " '184': 558,\n",
       " '185': 559,\n",
       " '186': 560,\n",
       " '1868': 561,\n",
       " '187': 562,\n",
       " '188': 563,\n",
       " '189': 564,\n",
       " '1890s': 565,\n",
       " '18\\\\/32': 566,\n",
       " '18th': 567,\n",
       " '18th-century': 568,\n",
       " '19': 569,\n",
       " '19-month': 570,\n",
       " '19-month-old': 571,\n",
       " '19.2': 572,\n",
       " '19.25': 573,\n",
       " '19.5': 574,\n",
       " '19.50': 575,\n",
       " '19.6': 576,\n",
       " '19.7': 577,\n",
       " '19.95': 578,\n",
       " '190': 579,\n",
       " '190-point': 580,\n",
       " '190.58': 581,\n",
       " '190.58-point': 582,\n",
       " '1900': 583,\n",
       " '1900s': 584,\n",
       " '1906': 585,\n",
       " '1908': 586,\n",
       " '191.75': 587,\n",
       " '192.5': 588,\n",
       " '1920s': 589,\n",
       " '1926': 590,\n",
       " '1929': 591,\n",
       " '193': 592,\n",
       " '193.3': 593,\n",
       " '1930': 594,\n",
       " '1930s': 595,\n",
       " '1932': 596,\n",
       " '1935': 597,\n",
       " '1939': 598,\n",
       " '1940s': 599,\n",
       " '1942': 600,\n",
       " '1947': 601,\n",
       " '1948': 602,\n",
       " '1949': 603,\n",
       " '1950': 604,\n",
       " '1950s': 605,\n",
       " '1951': 606,\n",
       " '1953': 607,\n",
       " '1955': 608,\n",
       " '1956': 609,\n",
       " '1957': 610,\n",
       " '1958': 611,\n",
       " '1959': 612,\n",
       " '196': 613,\n",
       " '1960': 614,\n",
       " '1960s': 615,\n",
       " '1961': 616,\n",
       " '1962': 617,\n",
       " '1963': 618,\n",
       " '1964': 619,\n",
       " '1965': 620,\n",
       " '1966': 621,\n",
       " '1967': 622,\n",
       " '1968': 623,\n",
       " '1969': 624,\n",
       " '197': 625,\n",
       " '1970': 626,\n",
       " '1970s': 627,\n",
       " '1971': 628,\n",
       " '1972': 629,\n",
       " '1973': 630,\n",
       " '1974': 631,\n",
       " '1975': 632,\n",
       " '1976': 633,\n",
       " '1977': 634,\n",
       " '1978': 635,\n",
       " '1979': 636,\n",
       " '1979-80': 637,\n",
       " '198': 638,\n",
       " '198,120,000': 639,\n",
       " '1980': 640,\n",
       " '1980s': 641,\n",
       " '1981': 642,\n",
       " '1982': 643,\n",
       " '1982-83': 644,\n",
       " '1983': 645,\n",
       " '1984': 646,\n",
       " '1985': 647,\n",
       " '1986': 648,\n",
       " '1987': 649,\n",
       " '1987-88': 650,\n",
       " '1988': 651,\n",
       " '1989': 652,\n",
       " '1989-A': 653,\n",
       " '1989A': 654,\n",
       " '1989B': 655,\n",
       " '199': 656,\n",
       " '1990': 657,\n",
       " '1990-2002': 658,\n",
       " '1990-model': 659,\n",
       " '1990s': 660,\n",
       " '1991': 661,\n",
       " '1992': 662,\n",
       " '1993': 663,\n",
       " '1993-2009': 664,\n",
       " '1994': 665,\n",
       " '1995': 666,\n",
       " '1996': 667,\n",
       " '1997': 668,\n",
       " '1998': 669,\n",
       " '1999': 670,\n",
       " '19\\\\/32': 671,\n",
       " '19th': 672,\n",
       " '19th-century': 673,\n",
       " '1:11': 674,\n",
       " '1\\\\/2': 675,\n",
       " '1\\\\/2-year': 676,\n",
       " '1\\\\/32': 677,\n",
       " '1\\\\/4': 678,\n",
       " '1\\\\/8': 679,\n",
       " '2': 680,\n",
       " '2,000': 681,\n",
       " '2,002': 682,\n",
       " '2,064': 683,\n",
       " '2,100': 684,\n",
       " '2,120': 685,\n",
       " '2,202,000': 686,\n",
       " '2,205,000': 687,\n",
       " '2,250,000': 688,\n",
       " '2,360': 689,\n",
       " '2,400': 690,\n",
       " '2,500': 691,\n",
       " '2,700': 692,\n",
       " '2-for-1': 693,\n",
       " '2-to-1': 694,\n",
       " '2.01': 695,\n",
       " '2.02': 696,\n",
       " '2.03': 697,\n",
       " '2.04': 698,\n",
       " '2.06': 699,\n",
       " '2.07': 700,\n",
       " '2.08': 701,\n",
       " '2.09': 702,\n",
       " '2.1': 703,\n",
       " '2.10': 704,\n",
       " '2.125': 705,\n",
       " '2.14': 706,\n",
       " '2.15': 707,\n",
       " '2.17': 708,\n",
       " '2.19': 709,\n",
       " '2.2': 710,\n",
       " '2.21': 711,\n",
       " '2.22': 712,\n",
       " '2.23': 713,\n",
       " '2.25': 714,\n",
       " '2.26': 715,\n",
       " '2.27': 716,\n",
       " '2.28': 717,\n",
       " '2.29': 718,\n",
       " '2.3': 719,\n",
       " '2.30': 720,\n",
       " '2.32': 721,\n",
       " '2.33': 722,\n",
       " '2.34': 723,\n",
       " '2.35': 724,\n",
       " '2.36': 725,\n",
       " '2.375': 726,\n",
       " '2.38': 727,\n",
       " '2.4': 728,\n",
       " '2.40': 729,\n",
       " '2.41': 730,\n",
       " '2.44': 731,\n",
       " '2.45': 732,\n",
       " '2.46': 733,\n",
       " '2.5': 734,\n",
       " '2.5-mile': 735,\n",
       " '2.50': 736,\n",
       " '2.51': 737,\n",
       " '2.53': 738,\n",
       " '2.56': 739,\n",
       " '2.57': 740,\n",
       " '2.58': 741,\n",
       " '2.6': 742,\n",
       " '2.60': 743,\n",
       " '2.61': 744,\n",
       " '2.62': 745,\n",
       " '2.625': 746,\n",
       " '2.63': 747,\n",
       " '2.65': 748,\n",
       " '2.66': 749,\n",
       " '2.68': 750,\n",
       " '2.69': 751,\n",
       " '2.7': 752,\n",
       " '2.70': 753,\n",
       " '2.73': 754,\n",
       " '2.74': 755,\n",
       " '2.75': 756,\n",
       " '2.77': 757,\n",
       " '2.79': 758,\n",
       " '2.8': 759,\n",
       " '2.80': 760,\n",
       " '2.82': 761,\n",
       " '2.85': 762,\n",
       " '2.87': 763,\n",
       " '2.875': 764,\n",
       " '2.88': 765,\n",
       " '2.9': 766,\n",
       " '2.90': 767,\n",
       " '2.95': 768,\n",
       " '20': 769,\n",
       " '20,000': 770,\n",
       " '20-stock': 771,\n",
       " '20-year': 772,\n",
       " '20-year-old': 773,\n",
       " '20.125': 774,\n",
       " '20.3': 775,\n",
       " '20.42': 776,\n",
       " '20.5': 777,\n",
       " '20.6': 778,\n",
       " '20.75': 779,\n",
       " '20.9': 780,\n",
       " '200': 781,\n",
       " '200,000': 782,\n",
       " '200,000-share': 783,\n",
       " '2000': 784,\n",
       " '2001': 785,\n",
       " '2002': 786,\n",
       " '2003': 787,\n",
       " '2003-2005': 788,\n",
       " '2003\\\\/2007': 789,\n",
       " '2004': 790,\n",
       " '2005': 791,\n",
       " '2006': 792,\n",
       " '2007': 793,\n",
       " '2008': 794,\n",
       " '2008-2009': 795,\n",
       " '2009': 796,\n",
       " '2010': 797,\n",
       " '2011': 798,\n",
       " '2012': 799,\n",
       " '2013': 800,\n",
       " '2014': 801,\n",
       " '2015': 802,\n",
       " '2016': 803,\n",
       " '2018': 804,\n",
       " '2019': 805,\n",
       " '2020': 806,\n",
       " '2023': 807,\n",
       " '205': 808,\n",
       " '206': 809,\n",
       " '207': 810,\n",
       " '208': 811,\n",
       " '208.7': 812,\n",
       " '209,000': 813,\n",
       " '20th': 814,\n",
       " '21': 815,\n",
       " '21.1': 816,\n",
       " '21.125': 817,\n",
       " '21.2': 818,\n",
       " '21.3': 819,\n",
       " '21.4': 820,\n",
       " '21.44': 821,\n",
       " '21.5': 822,\n",
       " '21.6': 823,\n",
       " '21.7': 824,\n",
       " '21.8': 825,\n",
       " '210': 826,\n",
       " '213': 827,\n",
       " '2149.3': 828,\n",
       " '215': 829,\n",
       " '216': 830,\n",
       " '217': 831,\n",
       " '219': 832,\n",
       " '21\\\\/32': 833,\n",
       " '21st': 834,\n",
       " '22': 835,\n",
       " '22,000': 836,\n",
       " '22.125': 837,\n",
       " '22.25': 838,\n",
       " '22.4': 839,\n",
       " '22.5': 840,\n",
       " '22.50': 841,\n",
       " '22.6': 842,\n",
       " '22.78': 843,\n",
       " '22.8': 844,\n",
       " '22.9': 845,\n",
       " '220': 846,\n",
       " '220,000': 847,\n",
       " '2200': 848,\n",
       " '221': 849,\n",
       " '222': 850,\n",
       " '224': 851,\n",
       " '224,070,000': 852,\n",
       " '224.1': 853,\n",
       " '225': 854,\n",
       " '226.3': 855,\n",
       " '229': 856,\n",
       " '22\\\\/32': 857,\n",
       " '22nd': 858,\n",
       " '23': 859,\n",
       " '23,000': 860,\n",
       " '23.1': 861,\n",
       " '23.2': 862,\n",
       " '23.25': 863,\n",
       " '23.5': 864,\n",
       " '23.625': 865,\n",
       " '23.7': 866,\n",
       " '23.8': 867,\n",
       " '23.9': 868,\n",
       " '230': 869,\n",
       " '231': 870,\n",
       " '231-191': 871,\n",
       " '232': 872,\n",
       " '232.3': 873,\n",
       " '234': 874,\n",
       " '235': 875,\n",
       " '235.2': 876,\n",
       " '237,960,000': 877,\n",
       " '238': 878,\n",
       " '23\\\\/32': 879,\n",
       " '24': 880,\n",
       " '24,000': 881,\n",
       " '24-hour': 882,\n",
       " '24-month': 883,\n",
       " '24.2': 884,\n",
       " '24.25': 885,\n",
       " '24.4': 886,\n",
       " '24.5': 887,\n",
       " '24.8': 888,\n",
       " '24.875': 889,\n",
       " '24.9': 890,\n",
       " '240': 891,\n",
       " '240,000': 892,\n",
       " '242': 893,\n",
       " '244': 894,\n",
       " '245': 895,\n",
       " '246.6': 896,\n",
       " '247': 897,\n",
       " '248': 898,\n",
       " '24th': 899,\n",
       " '25': 900,\n",
       " '25,000': 901,\n",
       " '25-year-old': 902,\n",
       " '25.2': 903,\n",
       " '25.3': 904,\n",
       " '25.4': 905,\n",
       " '25.5': 906,\n",
       " '25.6': 907,\n",
       " '25.8': 908,\n",
       " '25.875': 909,\n",
       " '250': 910,\n",
       " '250,000': 911,\n",
       " '251': 912,\n",
       " '252': 913,\n",
       " '253': 914,\n",
       " '254': 915,\n",
       " '255': 916,\n",
       " '256.6': 917,\n",
       " '257.8': 918,\n",
       " '258': 919,\n",
       " '2596.72': 920,\n",
       " '25\\\\/32': 921,\n",
       " '25th': 922,\n",
       " '26': 923,\n",
       " '26,000': 924,\n",
       " '26-week': 925,\n",
       " '26-year-old': 926,\n",
       " '26.1': 927,\n",
       " '26.23': 928,\n",
       " '26.3': 929,\n",
       " '26.5': 930,\n",
       " '26.50': 931,\n",
       " '26.7': 932,\n",
       " '26.9': 933,\n",
       " '260': 934,\n",
       " '2603.48': 935,\n",
       " '2638.73': 936,\n",
       " '264': 937,\n",
       " '2643.65': 938,\n",
       " '2645.08': 939,\n",
       " '265': 940,\n",
       " '2653.28': 941,\n",
       " '2659.22': 942,\n",
       " '266': 943,\n",
       " '266.2': 944,\n",
       " '266.66': 945,\n",
       " '2662.91': 946,\n",
       " '267': 947,\n",
       " '268': 948,\n",
       " '2683.20': 949,\n",
       " '2689.14': 950,\n",
       " '269': 951,\n",
       " '26\\\\/32': 952,\n",
       " '27': 953,\n",
       " '27,000': 954,\n",
       " '27-year-old': 955,\n",
       " '27.1': 956,\n",
       " '27.5': 957,\n",
       " '27.6': 958,\n",
       " '27.7': 959,\n",
       " '27.8': 960,\n",
       " '27.9': 961,\n",
       " '270': 962,\n",
       " '271': 963,\n",
       " '273': 964,\n",
       " '274': 965,\n",
       " '275': 966,\n",
       " '275,000': 967,\n",
       " '276,334': 968,\n",
       " '276.8': 969,\n",
       " '278': 970,\n",
       " '279': 971,\n",
       " '2791.41': 972,\n",
       " '28': 973,\n",
       " '28,000': 974,\n",
       " '28.4': 975,\n",
       " '28.5': 976,\n",
       " '28.6': 977,\n",
       " '28.7': 978,\n",
       " '28.75': 979,\n",
       " '280': 980,\n",
       " '282': 981,\n",
       " '283.7': 982,\n",
       " '283.8': 983,\n",
       " '285': 984,\n",
       " '286': 985,\n",
       " '287': 986,\n",
       " '288': 987,\n",
       " '289': 988,\n",
       " '28\\\\/32': 989,\n",
       " '29': 990,\n",
       " '29-year-old': 991,\n",
       " '29.4': 992,\n",
       " '29.6': 993,\n",
       " '29.7': 994,\n",
       " '290': 995,\n",
       " '293': 996,\n",
       " '294': 997,\n",
       " '29\\\\/32': 998,\n",
       " '2\\\\/32': 999,\n",
       " ...}"
      ]
     },
     "metadata": {},
     "execution_count": 189
    }
   ],
   "source": [
    "vocabulary_with_index = {}\n",
    "\n",
    "for i, word in enumerate(sorted(vocabulary)):\n",
    "    vocabulary_with_index[word] = i\n",
    "\n",
    "vocabulary_with_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "N\n",
      "\n",
      "of\tIN\n",
      "\n",
      "syndication\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Under\tIN\n",
      "\n",
      "current\tJJ\n",
      "\n",
      "rules\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "even\tRB\n",
      "\n",
      "when\tWRB\n",
      "\n",
      "a\tDT\n",
      "\n",
      "network\tNN\n",
      "\n",
      "fares\tVBZ\n",
      "\n",
      "well\tRB\n",
      "\n",
      "with\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "100%-owned\tJJ\n",
      "\n",
      "series\tNN\n",
      "\n",
      "--\t:\n",
      "\n",
      "ABC\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "for\tIN\n",
      "\n",
      "example\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "made\tVBD\n",
      "\n",
      "a\tDT\n",
      "\n",
      "killing\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "broadcasting\tVBG\n",
      "\n",
      "its\tPRP$\n",
      "\n",
      "popular\tJJ\n",
      "\n",
      "crime\\/comedy\tNN\n",
      "\n",
      "``\t``\n",
      "\n",
      "Moonlighting\tNNP\n",
      "\n",
      "''\t''\n",
      "\n",
      "--\t:\n",
      "\n",
      "it\tPRP\n",
      "\n",
      "is\tVBZ\n",
      "\n",
      "n't\tRB\n",
      "\n",
      "allowed\tVBN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "share\tVB\n",
      "\n",
      "in\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "continuing\tVBG\n",
      "\n",
      "proceeds\tNNS\n",
      "\n",
      "when\tWRB\n",
      "\n",
      "the\tDT\n",
      "\n",
      "reruns\tNNS\n",
      "\n",
      "are\tVBP\n",
      "\n",
      "sold\tVBN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "local\tJJ\n",
      "\n",
      "stations\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Instead\tRB\n",
      "\n",
      ",\t,\n",
      "\n",
      "ABC\tNNP\n",
      "\n",
      "will\tMD\n",
      "\n",
      "have\tVB\n",
      "\n",
      "to\tTO\n",
      "\n",
      "sell\tVB\n",
      "\n",
      "off\tRP\n",
      "\n",
      "the\tDT\n",
      "\n",
      "rights\tNNS\n",
      "\n",
      "for\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "one-time\tJJ\n",
      "\n",
      "fee\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "networks\tNNS\n",
      "\n",
      "admit\tVBP\n",
      "\n",
      "that\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "chances\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "getting\tVBG\n",
      "\n",
      "the\tDT\n",
      "\n",
      "relief\tNN\n",
      "\n",
      "they\tPRP\n",
      "\n",
      "want\tVBP\n",
      "\n",
      "are\tVBP\n",
      "\n",
      "slim\tJJ\n",
      "\n",
      "--\t:\n",
      "\n",
      "for\tIN\n",
      "\n",
      "several\tJJ\n",
      "\n",
      "years\tNNS\n",
      "\n",
      "at\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "least\tJJS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Six\tCD\n",
      "\n",
      "years\tNNS\n",
      "\n",
      "ago\tIN\n",
      "\n",
      "they\tPRP\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "tantalizingly\tRB\n",
      "\n",
      "close\tJJ\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "Reagan-era\tNNP\n",
      "\n",
      "Federal\tNNP\n",
      "\n",
      "Communications\tNNPS\n",
      "\n",
      "Commission\tNNP\n",
      "\n",
      "had\tVBD\n",
      "\n",
      "ruled\tVBN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "favor\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "killing\tVBG\n",
      "\n",
      "most\tJJS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "rules\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Various\tJJ\n",
      "\n",
      "evidence\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "including\tVBG\n",
      "\n",
      "a\tDT\n",
      "\n",
      "Brookings\tNNP\n",
      "\n",
      "Institution\tNNP\n",
      "\n",
      "study\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "some\tDT\n",
      "\n",
      "800\tCD\n",
      "\n",
      "series\tNN\n",
      "\n",
      "that\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "networks\tNNS\n",
      "\n",
      "had\tVBD\n",
      "\n",
      "aired\tVBN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "had\tVBD\n",
      "\n",
      "partly\tRB\n",
      "\n",
      "owned\tVBN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "1960s\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "showed\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "networks\tNNS\n",
      "\n",
      "did\tVBD\n",
      "\n",
      "n't\tRB\n",
      "\n",
      "wield\tVB\n",
      "\n",
      "undue\tJJ\n",
      "\n",
      "control\tNN\n",
      "\n",
      "over\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "studios\tNNS\n",
      "\n",
      "as\tIN\n",
      "\n",
      "had\tVBD\n",
      "\n",
      "been\tVBN\n",
      "\n",
      "alleged\tVBN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "But\tCC\n",
      "\n",
      "just\tRB\n",
      "\n",
      "eight\tCD\n",
      "\n",
      "days\tNNS\n",
      "\n",
      "before\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "rules\tNNS\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "to\tTO\n",
      "\n",
      "die\tVB\n",
      "\n",
      ",\t,\n",
      "\n",
      "former\tJJ\n",
      "\n",
      "President\tNNP\n",
      "\n",
      "Ronald\tNNP\n",
      "\n",
      "Reagan\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "a\tDT\n",
      "\n",
      "one-time\tJJ\n",
      "\n",
      "actor\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "intervened\tVBD\n",
      "\n",
      "on\tIN\n",
      "\n",
      "behalf\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "Hollywood\tNNP\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "FCC\tNNP\n",
      "\n",
      "effort\tNN\n",
      "\n",
      "collapsed\tVBD\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "networks\tNNS\n",
      "\n",
      "and\tCC\n",
      "\n",
      "studios\tNNS\n",
      "\n",
      "have\tVBP\n",
      "\n",
      "bickered\tVBN\n",
      "\n",
      "ever\tRB\n",
      "\n",
      "since\tIN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Network\tNN\n",
      "\n",
      "officials\tNNS\n",
      "\n",
      "involved\tVBN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "studio\tNN\n",
      "\n",
      "talks\tNNS\n",
      "\n",
      "may\tMD\n",
      "\n",
      "hope\tVB\n",
      "\n",
      "the\tDT\n",
      "\n",
      "foreign\tJJ\n",
      "\n",
      "influx\tNN\n",
      "\n",
      "builds\tVBZ\n",
      "\n",
      "more\tJJR\n",
      "\n",
      "support\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "Washington\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "but\tCC\n",
      "\n",
      "that\tDT\n",
      "\n",
      "seems\tVBZ\n",
      "\n",
      "unlikely\tJJ\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "In\tIN\n",
      "\n",
      "Congress\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "issue\tNN\n",
      "\n",
      "falters\tVBZ\n",
      "\n",
      ":\t:\n",
      "\n",
      "It\tPRP\n",
      "\n",
      "'s\tVBZ\n",
      "\n",
      "about\tIN\n",
      "\n",
      "money\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "not\tRB\n",
      "\n",
      "program\tNN\n",
      "\n",
      "quality\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "and\tCC\n",
      "\n",
      "Hollywood\tNNP\n",
      "\n",
      "has\tVBZ\n",
      "\n",
      "lots\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "clout\tNN\n",
      "\n",
      "given\tVBN\n",
      "\n",
      "its\tPRP$\n",
      "\n",
      "fund\tNN\n",
      "\n",
      "raising\tNN\n",
      "\n",
      "for\tIN\n",
      "\n",
      "senators\tNNS\n",
      "\n",
      "and\tCC\n",
      "\n",
      "representatives\tNNS\n",
      "\n",
      "overseeing\tVBG\n",
      "\n",
      "the\tDT\n",
      "\n",
      "issue\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "A\tDT\n",
      "\n",
      "spokesman\tNN\n",
      "\n",
      "for\tIN\n",
      "\n",
      "Rep.\tNNP\n",
      "\n",
      "Edward\tNNP\n",
      "\n",
      "J.\tNNP\n",
      "\n",
      "Markey\tNNP\n",
      "\n",
      "(\t(\n",
      "\n",
      "D-Mass.\tNNP\n",
      "\n",
      ")\t)\n",
      "\n",
      ",\t,\n",
      "\n",
      "who\tWP\n",
      "\n",
      "heads\tVBZ\n",
      "\n",
      "a\tDT\n",
      "\n",
      "subcommittee\tNN\n",
      "\n",
      "that\tWDT\n",
      "\n",
      "oversees\tVBZ\n",
      "\n",
      "the\tDT\n",
      "\n",
      "FCC\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "says\tVBZ\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Markey\tNNP\n",
      "\n",
      "feels\tVBZ\n",
      "\n",
      "``\t``\n",
      "\n",
      "the\tDT\n",
      "\n",
      "world\tNN\n",
      "\n",
      "has\tVBZ\n",
      "\n",
      "been\tVBN\n",
      "\n",
      "forever\tRB\n",
      "\n",
      "changed\tVBN\n",
      "\n",
      "by\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "Sony-Columbia\tNNP\n",
      "\n",
      "deal\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "''\t''\n",
      "\n",
      "\n",
      "\n",
      "But\tCC\n",
      "\n",
      "he\tPRP\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Markey\tNNP\n",
      "\n",
      "hopes\tVBZ\n",
      "\n",
      "this\tDT\n",
      "\n",
      "pushes\tVBZ\n",
      "\n",
      "the\tDT\n",
      "\n",
      "networks\tNNS\n",
      "\n",
      "and\tCC\n",
      "\n",
      "studios\tNNS\n",
      "\n",
      "to\tTO\n",
      "\n",
      "work\tVB\n",
      "\n",
      "it\tPRP\n",
      "\n",
      "out\tRP\n",
      "\n",
      "on\tIN\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "own\tJJ\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "And\tCC\n",
      "\n",
      "at\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "FCC\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "meanwhile\tRB\n",
      "\n",
      ",\t,\n",
      "\n",
      "new\tJJ\n",
      "\n",
      "Chairman\tNNP\n",
      "\n",
      "Alfred\tNNP\n",
      "\n",
      "C.\tNNP\n",
      "\n",
      "Sikes\tNNP\n",
      "\n",
      "has\tVBZ\n",
      "\n",
      "said\tVBN\n",
      "\n",
      "he\tPRP\n",
      "\n",
      "wants\tVBZ\n",
      "\n",
      "the\tDT\n",
      "\n",
      "two\tCD\n",
      "\n",
      "sides\tNNS\n",
      "\n",
      "to\tTO\n",
      "\n",
      "hammer\tVB\n",
      "\n",
      "out\tRP\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "own\tJJ\n",
      "\n",
      "plan\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Recognition\tNNP\n",
      "\n",
      "Equipment\tNNP\n",
      "\n",
      "Inc.\tNNP\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "it\tPRP\n",
      "\n",
      "settled\tVBD\n",
      "\n",
      "a\tDT\n",
      "\n",
      "civil\tJJ\n",
      "\n",
      "action\tNN\n",
      "\n",
      "filed\tVBN\n",
      "\n",
      "against\tIN\n",
      "\n",
      "it\tPRP\n",
      "\n",
      "by\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "federal\tJJ\n",
      "\n",
      "government\tNN\n",
      "\n",
      "on\tIN\n",
      "\n",
      "behalf\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "U.S.\tNNP\n",
      "\n",
      "Postal\tNNP\n",
      "\n",
      "Service\tNNP\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      "sued\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "April\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "seeking\tVBG\n",
      "\n",
      "$\t$\n",
      "\n",
      "23,000\tCD\n",
      "\n",
      "and\tCC\n",
      "\n",
      "other\tJJ\n",
      "\n",
      "unspecified\tJJ\n",
      "\n",
      "damages\tNNS\n",
      "\n",
      "related\tVBN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "an\tDT\n",
      "\n",
      "alleged\tJJ\n",
      "\n",
      "contract-steering\tJJ\n",
      "\n",
      "scheme\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "suit\tNN\n",
      "\n",
      "named\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "former\tJJ\n",
      "\n",
      "chief\tNN\n",
      "\n",
      "executive\tNN\n",
      "\n",
      "officer\tNN\n",
      "\n",
      "William\tNNP\n",
      "\n",
      "G.\tNNP\n",
      "\n",
      "Moore\tNNP\n",
      "\n",
      "Jr.\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "former\tJJ\n",
      "\n",
      "vice\tNN\n",
      "\n",
      "president\tNN\n",
      "\n",
      "Robert\tNNP\n",
      "\n",
      "W.\tNNP\n",
      "\n",
      "Reedy\tNNP\n",
      "\n",
      "and\tCC\n",
      "\n",
      "five\tCD\n",
      "\n",
      "defendants\tNNS\n",
      "\n",
      "who\tWP\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "n't\tRB\n",
      "\n",
      "part\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "suit\tNN\n",
      "\n",
      "charged\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "defendants\tNNS\n",
      "\n",
      "with\tIN\n",
      "\n",
      "causing\tVBG\n",
      "\n",
      "Peter\tNNP\n",
      "\n",
      "E.\tNNP\n",
      "\n",
      "Voss\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "an\tDT\n",
      "\n",
      "ex-member\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "Postal\tNNP\n",
      "\n",
      "Service\tNNP\n",
      "\n",
      "board\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "governors\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "to\tTO\n",
      "\n",
      "accept\tVB\n",
      "\n",
      "$\t$\n",
      "\n",
      "23,000\tCD\n",
      "\n",
      "in\tIN\n",
      "\n",
      "bribes\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "kickbacks\tNNS\n",
      "\n",
      "and\tCC\n",
      "\n",
      "gratuities\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Voss\tNNP\n",
      "\n",
      "was\tVBD\n",
      "\n",
      "previously\tRB\n",
      "\n",
      "sentenced\tVBN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "four\tCD\n",
      "\n",
      "years\tNNS\n",
      "\n",
      "in\tIN\n",
      "\n",
      "prison\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "fined\tVBN\n",
      "\n",
      "$\t$\n",
      "\n",
      "11,000\tCD\n",
      "\n",
      "for\tIN\n",
      "\n",
      "his\tPRP$\n",
      "\n",
      "role\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "scheme\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "In\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "agreement\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "Recognition\tNN\n",
      "\n",
      "agreed\tVBD\n",
      "\n",
      "to\tTO\n",
      "\n",
      "pay\tVB\n",
      "\n",
      "the\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      "$\t$\n",
      "\n",
      "20,000\tCD\n",
      "\n",
      "in\tIN\n",
      "\n",
      "return\tNN\n",
      "\n",
      "for\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "release\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "all\tDT\n",
      "\n",
      "claims\tNNS\n",
      "\n",
      "against\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Moore\tNNP\n",
      "\n",
      "and\tCC\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Reedy\tNNP\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "five\tCD\n",
      "\n",
      "additional\tJJ\n",
      "\n",
      "defendants\tNNS\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "n't\tRB\n",
      "\n",
      "parties\tNNS\n",
      "\n",
      "to\tTO\n",
      "\n",
      "the\tDT\n",
      "\n",
      "settlement\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "A\tDT\n",
      "\n",
      "trial\tNN\n",
      "\n",
      "on\tIN\n",
      "\n",
      "criminal\tJJ\n",
      "\n",
      "allegations\tNNS\n",
      "\n",
      "against\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "the\tDT\n",
      "\n",
      "same\tJJ\n",
      "\n",
      "two\tCD\n",
      "\n",
      "former\tJJ\n",
      "\n",
      "executives\tNNS\n",
      "\n",
      "began\tVBD\n",
      "\n",
      "Sept.\tNNP\n",
      "\n",
      "27\tCD\n",
      "\n",
      "in\tIN\n",
      "\n",
      "federal\tJJ\n",
      "\n",
      "court\tNN\n",
      "\n",
      "for\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "District\tNNP\n",
      "\n",
      "of\tIN\n",
      "\n",
      "Columbia\tNNP\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "They\tPRP\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "indicted\tVBN\n",
      "\n",
      "last\tJJ\n",
      "\n",
      "October\tNNP\n",
      "\n",
      "on\tIN\n",
      "\n",
      "charges\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "fraud\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "theft\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "conspiracy\tNN\n",
      "\n",
      "related\tVBN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "an\tDT\n",
      "\n",
      "effort\tNN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "win\tVB\n",
      "\n",
      "$\t$\n",
      "\n",
      "400\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      "in\tIN\n",
      "\n",
      "Postal\tNNP\n",
      "\n",
      "Service\tNNP\n",
      "\n",
      "equipment\tNN\n",
      "\n",
      "contracts\tNNS\n",
      "\n",
      "by\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "maker\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "data\tNNS\n",
      "\n",
      "management\tNN\n",
      "\n",
      "equipment\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "its\tPRP$\n",
      "\n",
      "executives\tNNS\n",
      "\n",
      "deny\tVBP\n",
      "\n",
      "the\tDT\n",
      "\n",
      "charges\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "In\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "related\tVBN\n",
      "\n",
      "development\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "Recognition\tNNP\n",
      "\n",
      "Equipment\tNNP\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "Postal\tNNP\n",
      "\n",
      "Service\tNNP\n",
      "\n",
      "has\tVBZ\n",
      "\n",
      "barred\tVBN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      "from\tIN\n",
      "\n",
      "bidding\tVBG\n",
      "\n",
      "on\tIN\n",
      "\n",
      "postal\tJJ\n",
      "\n",
      "contracts\tNNS\n",
      "\n",
      "for\tIN\n",
      "\n",
      "an\tDT\n",
      "\n",
      "additional\tJJ\n",
      "\n",
      "120\tCD\n",
      "\n",
      "days\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "Postal\tNNP\n",
      "\n",
      "Service\tNNP\n",
      "\n",
      "originally\tRB\n",
      "\n",
      "suspended\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      "Oct.\tNNP\n",
      "\n",
      "7\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "1988\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "and\tCC\n",
      "\n",
      "has\tVBZ\n",
      "\n",
      "been\tVBN\n",
      "\n",
      "renewing\tVBG\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ban\tNN\n",
      "\n",
      "ever\tRB\n",
      "\n",
      "since\tIN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "it\tPRP\n",
      "\n",
      "will\tMD\n",
      "\n",
      "continue\tVB\n",
      "\n",
      "to\tTO\n",
      "\n",
      "pursue\tVB\n",
      "\n",
      "a\tDT\n",
      "\n",
      "lifting\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "suspension\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Intel\tNNP\n",
      "\n",
      "Corp.\tNNP\n",
      "\n",
      "reported\tVBD\n",
      "\n",
      "a\tDT\n",
      "\n",
      "50\tCD\n",
      "\n",
      "%\tNN\n",
      "\n",
      "drop\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "third-quarter\tNN\n",
      "\n",
      "net\tNN\n",
      "\n",
      "income\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "partly\tRB\n",
      "\n",
      "because\tIN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "one-time\tJJ\n",
      "\n",
      "charge\tNN\n",
      "\n",
      "for\tIN\n",
      "\n",
      "discontinued\tVBN\n",
      "\n",
      "operations\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "big\tJJ\n",
      "\n",
      "semiconductor\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "computer\tNN\n",
      "\n",
      "maker\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "it\tPRP\n",
      "\n",
      "had\tVBD\n",
      "\n",
      "net\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "72\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "or\tCC\n",
      "\n",
      "38\tCD\n",
      "\n",
      "cents\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "down\tIN\n",
      "\n",
      "50\tCD\n",
      "\n",
      "%\tNN\n",
      "\n",
      "from\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "142.7\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "or\tCC\n",
      "\n",
      "78\tCD\n",
      "\n",
      "cents\tNNS\n",
      "\n",
      "a\tDT\n",
      "\n",
      "share\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "lower\tJJR\n",
      "\n",
      "net\tNN\n",
      "\n",
      "included\tVBD\n",
      "\n",
      "a\tDT\n",
      "\n",
      "charge\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "35\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "equal\tJJ\n",
      "\n",
      "to\tTO\n",
      "\n",
      "12\tCD\n",
      "\n",
      "cents\tNNS\n",
      "\n",
      "a\tDT\n",
      "\n",
      "share\tNN\n",
      "\n",
      "on\tIN\n",
      "\n",
      "an\tDT\n",
      "\n",
      "after-tax\tJJ\n",
      "\n",
      "basis\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "for\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "cost\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "abandoning\tVBG\n",
      "\n",
      "a\tDT\n",
      "\n",
      "computer-systems\tNNS\n",
      "\n",
      "joint\tNN\n",
      "\n",
      "venture\tNN\n",
      "\n",
      "with\tIN\n",
      "\n",
      "Siemens\tNNP\n",
      "\n",
      "AG\tNNP\n",
      "\n",
      "of\tIN\n",
      "\n",
      "West\tNNP\n",
      "\n",
      "Germany\tNNP\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Earning\tNN\n",
      "\n",
      "also\tRB\n",
      "\n",
      "fell\tVBD\n",
      "\n",
      "from\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "year-ago\tJJ\n",
      "\n",
      "period\tNN\n",
      "\n",
      "because\tIN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "slowing\tVBG\n",
      "\n",
      "microchip\tNN\n",
      "\n",
      "demand\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Sales\tNNS\n",
      "\n",
      "amounted\tVBD\n",
      "\n",
      "to\tTO\n",
      "\n",
      "$\t$\n",
      "\n",
      "771.4\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "down\tIN\n",
      "\n",
      "1.7\tCD\n",
      "\n",
      "%\tNN\n",
      "\n",
      "from\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "784.9\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Intel\tNNP\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "stock\tNN\n",
      "\n",
      "rose\tVBD\n",
      "\n",
      "in\tIN\n",
      "\n",
      "early\tJJ\n",
      "\n",
      "over-the-counter\tNN\n",
      "\n",
      "trading\tNN\n",
      "\n",
      "Friday\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "as\tIN\n",
      "\n",
      "investors\tNNS\n",
      "\n",
      "appeared\tVBD\n",
      "\n",
      "relieved\tVBN\n",
      "\n",
      "that\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "income\tNN\n",
      "\n",
      "from\tIN\n",
      "\n",
      "continuing\tVBG\n",
      "\n",
      "operations\tNNS\n",
      "\n",
      "was\tVBD\n",
      "\n",
      "only\tRB\n",
      "\n",
      "slightly\tRB\n",
      "\n",
      "below\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "second\tJJ\n",
      "\n",
      "quarter\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "earnings\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "99.3\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "or\tCC\n",
      "\n",
      "53\tCD\n",
      "\n",
      "cents\tNNS\n",
      "\n",
      "a\tDT\n",
      "\n",
      "share\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "and\tCC\n",
      "\n",
      "that\tIN\n",
      "\n",
      "sales\tNNS\n",
      "\n",
      "actually\tRB\n",
      "\n",
      "exceeded\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "$\t$\n",
      "\n",
      "747.3\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      "for\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "second\tJJ\n",
      "\n",
      "period\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "But\tCC\n",
      "\n",
      "Intel\tNNP\n",
      "\n",
      "later\tRB\n",
      "\n",
      "succumbed\tVBD\n",
      "\n",
      "to\tTO\n",
      "\n",
      "the\tDT\n",
      "\n",
      "stock\tNN\n",
      "\n",
      "market\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "plunge\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "closing\tVBG\n",
      "\n",
      "at\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "31.75\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "down\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "2.125\tCD\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "In\tIN\n",
      "\n",
      "August\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "Intel\tNNP\n",
      "\n",
      "warned\tVBD\n",
      "\n",
      "that\tIN\n",
      "\n",
      "third-quarter\tNN\n",
      "\n",
      "earnings\tNNS\n",
      "\n",
      "might\tMD\n",
      "\n",
      "be\tVB\n",
      "\n",
      "``\t``\n",
      "\n",
      "flat\tJJ\n",
      "\n",
      "to\tTO\n",
      "\n",
      "down\tJJ\n",
      "\n",
      "''\t''\n",
      "\n",
      "from\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "previous\tJJ\n",
      "\n",
      "period\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "because\tIN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "slowing\tVBG\n",
      "\n",
      "sales\tNNS\n",
      "\n",
      "growth\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "its\tPRP$\n",
      "\n",
      "80386\tCD\n",
      "\n",
      "microprocessor\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "start-up\tJJ\n",
      "\n",
      "costs\tNNS\n",
      "\n",
      "associated\tVBN\n",
      "\n",
      "with\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "line\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "computers\tNNS\n",
      "\n",
      "and\tCC\n",
      "\n",
      "costs\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "preparing\tVBG\n",
      "\n",
      "for\tIN\n",
      "\n",
      "mass\tNN\n",
      "\n",
      "shipments\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "company\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "new\tJJ\n",
      "\n",
      "80486\tCD\n",
      "\n",
      "chip\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "current\tJJ\n",
      "\n",
      "quarter\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "On\tIN\n",
      "\n",
      "Friday\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "Andrew\tNNP\n",
      "\n",
      "S.Grove\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "Intel\tNNP\n",
      "\n",
      "president\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "chief\tNN\n",
      "\n",
      "executive\tNN\n",
      "\n",
      "officer\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "``\t``\n",
      "\n",
      "Intel\tNNP\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "business\tNN\n",
      "\n",
      "is\tVBZ\n",
      "\n",
      "strong\tJJ\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Our\tPRP$\n",
      "\n",
      "bookings\tNNS\n",
      "\n",
      "improved\tVBD\n",
      "\n",
      "as\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "quarter\tNN\n",
      "\n",
      "progressed\tVBD\n",
      "\n",
      "and\tCC\n",
      "\n",
      "September\tNNP\n",
      "\n",
      "was\tVBD\n",
      "\n",
      "especially\tRB\n",
      "\n",
      "good\tJJ\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "For\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "full\tJJ\n",
      "\n",
      "quarter\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "our\tPRP$\n",
      "\n",
      "bookings\tNNS\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "higher\tJJR\n",
      "\n",
      "than\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "previous\tJJ\n",
      "\n",
      "quarter\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "and\tCC\n",
      "\n",
      "our\tPRP$\n",
      "\n",
      "book-to-bill\tNN\n",
      "\n",
      "ratio\tNN\n",
      "\n",
      "exceeded\tVBD\n",
      "\n",
      "1.0\tCD\n",
      "\n",
      ".\t.\n",
      "\n",
      "''\t''\n",
      "\n",
      "\n",
      "\n",
      "For\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "nine-month\tJJ\n",
      "\n",
      "period\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "Intel\tNNP\n",
      "\n",
      "reported\tVBD\n",
      "\n",
      "net\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "268.3\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "or\tCC\n",
      "\n",
      "$\t$\n",
      "\n",
      "1.43\tCD\n",
      "\n",
      "a\tDT\n",
      "\n",
      "share\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "down\tIN\n",
      "\n",
      "27\tCD\n",
      "\n",
      "%\tNN\n",
      "\n",
      "from\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "367.1\tCD\n",
      "\n",
      "million\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "or\tCC\n",
      "\n",
      "$\t$\n",
      "\n",
      "2.05\tCD\n",
      "\n",
      "a\tDT\n",
      "\n",
      "share\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Revenue\tNN\n",
      "\n",
      "amounted\tVBD\n",
      "\n",
      "to\tTO\n",
      "\n",
      "$\t$\n",
      "\n",
      "2.23\tCD\n",
      "\n",
      "billion\tCD\n",
      "\n",
      ",\t,\n",
      "\n",
      "up\tIN\n",
      "\n",
      "slightly\tRB\n",
      "\n",
      "from\tIN\n",
      "\n",
      "$\t$\n",
      "\n",
      "2.15\tCD\n",
      "\n",
      "billion\tCD\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Walter\tNNP\n",
      "\n",
      "Sisulu\tNNP\n",
      "\n",
      "and\tCC\n",
      "\n",
      "the\tDT\n",
      "\n",
      "African\tNNP\n",
      "\n",
      "National\tNNP\n",
      "\n",
      "Congress\tNNP\n",
      "\n",
      "came\tVBD\n",
      "\n",
      "home\tNN\n",
      "\n",
      "yesterday\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "After\tIN\n",
      "\n",
      "26\tCD\n",
      "\n",
      "years\tNNS\n",
      "\n",
      "in\tIN\n",
      "\n",
      "prison\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Sisulu\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "77-year-old\tJJ\n",
      "\n",
      "former\tJJ\n",
      "\n",
      "secretary-general\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "liberation\tNN\n",
      "\n",
      "movement\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "was\tVBD\n",
      "\n",
      "dropped\tVBN\n",
      "\n",
      "off\tRP\n",
      "\n",
      "at\tIN\n",
      "\n",
      "his\tPRP$\n",
      "\n",
      "house\tNN\n",
      "\n",
      "by\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "prison\tNN\n",
      "\n",
      "services\tNNS\n",
      "\n",
      "'\tPOS\n",
      "\n",
      "van\tNN\n",
      "\n",
      "just\tRB\n",
      "\n",
      "as\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "sun\tNN\n",
      "\n",
      "was\tVBD\n",
      "\n",
      "coming\tVBG\n",
      "\n",
      "up\tIN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "At\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "same\tJJ\n",
      "\n",
      "time\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "six\tCD\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "colleagues\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "five\tCD\n",
      "\n",
      "of\tIN\n",
      "\n",
      "whom\tWP\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "arrested\tVBN\n",
      "\n",
      "with\tIN\n",
      "\n",
      "him\tPRP\n",
      "\n",
      "in\tIN\n",
      "\n",
      "1963\tCD\n",
      "\n",
      "and\tCC\n",
      "\n",
      "sentenced\tVBN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "life\tNN\n",
      "\n",
      "imprisonment\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "reunited\tVBN\n",
      "\n",
      "with\tIN\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "families\tNNS\n",
      "\n",
      "at\tIN\n",
      "\n",
      "various\tJJ\n",
      "\n",
      "places\tNNS\n",
      "\n",
      "around\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "country\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "And\tCC\n",
      "\n",
      "as\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "graying\tVBG\n",
      "\n",
      "men\tNNS\n",
      "\n",
      "returned\tVBD\n",
      "\n",
      "to\tTO\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "homes\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "outlawed\tVBN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "South\tNNP\n",
      "\n",
      "Africa\tNNP\n",
      "\n",
      "since\tIN\n",
      "\n",
      "1960\tCD\n",
      "\n",
      "and\tCC\n",
      "\n",
      "still\tRB\n",
      "\n",
      "considered\tVBN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "be\tVB\n",
      "\n",
      "the\tDT\n",
      "\n",
      "chief\tNN\n",
      "\n",
      "public\tJJ\n",
      "\n",
      "enemy\tNN\n",
      "\n",
      "by\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "white\tJJ\n",
      "\n",
      "government\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "defiantly\tRB\n",
      "\n",
      "returned\tVBD\n",
      "\n",
      "to\tTO\n",
      "\n",
      "the\tDT\n",
      "\n",
      "streets\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "country\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "black\tJJ\n",
      "\n",
      "townships\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "A\tDT\n",
      "\n",
      "huge\tJJ\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "flag\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "with\tIN\n",
      "\n",
      "black\tJJ\n",
      "\n",
      ",\t,\n",
      "\n",
      "green\tJJ\n",
      "\n",
      "and\tCC\n",
      "\n",
      "gold\tJJ\n",
      "\n",
      "stripes\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "was\tVBD\n",
      "\n",
      "hoisted\tVBN\n",
      "\n",
      "over\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "rickety\tJJ\n",
      "\n",
      "gate\tNN\n",
      "\n",
      "at\tIN\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Sisulu\tNNP\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "modest\tJJ\n",
      "\n",
      "house\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "while\tIN\n",
      "\n",
      "on\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "street\tNN\n",
      "\n",
      "out\tIN\n",
      "\n",
      "front\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "boys\tNNS\n",
      "\n",
      "displayed\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "colors\tNNS\n",
      "\n",
      "on\tIN\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "shirts\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "caps\tNNS\n",
      "\n",
      "and\tCC\n",
      "\n",
      "scarves\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "At\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "small\tJJ\n",
      "\n",
      "four-room\tJJ\n",
      "\n",
      "home\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "Elias\tNNP\n",
      "\n",
      "Motsoaledi\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "a\tDT\n",
      "\n",
      "leading\tVBG\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "unionist\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "a\tDT\n",
      "\n",
      "former\tJJ\n",
      "\n",
      "commander\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "group\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "armed\tVBN\n",
      "\n",
      "wing\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "Umkhonto\tNNP\n",
      "\n",
      "we\tPRP\n",
      "\n",
      "Sizwe\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "well-wishers\tNNS\n",
      "\n",
      "stuck\tVBD\n",
      "\n",
      "little\tJJ\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "flags\tNNS\n",
      "\n",
      "in\tIN\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "hair\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "a\tDT\n",
      "\n",
      "man\tNN\n",
      "\n",
      "tooted\tVBD\n",
      "\n",
      "on\tIN\n",
      "\n",
      "an\tDT\n",
      "\n",
      "antelope\tNN\n",
      "\n",
      "horn\tNN\n",
      "\n",
      "wrapped\tVBN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "ribbons\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "``\t``\n",
      "\n",
      "I\tPRP\n",
      "\n",
      "am\tVBP\n",
      "\n",
      "happy\tJJ\n",
      "\n",
      "to\tTO\n",
      "\n",
      "see\tVB\n",
      "\n",
      "the\tDT\n",
      "\n",
      "spirit\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "people\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "''\t''\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Sisulu\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "looking\tVBG\n",
      "\n",
      "dapper\tJJ\n",
      "\n",
      "in\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "new\tJJ\n",
      "\n",
      "gray\tJJ\n",
      "\n",
      "suit\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "As\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "crowd\tNN\n",
      "\n",
      "outside\tIN\n",
      "\n",
      "his\tPRP$\n",
      "\n",
      "home\tNN\n",
      "\n",
      "shouted\tVBD\n",
      "\n",
      "``\t``\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "''\t''\n",
      "\n",
      "the\tDT\n",
      "\n",
      "old\tJJ\n",
      "\n",
      "man\tNN\n",
      "\n",
      "shot\tVBD\n",
      "\n",
      "his\tPRP$\n",
      "\n",
      "fists\tNNS\n",
      "\n",
      "into\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "air\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "``\t``\n",
      "\n",
      "I\tPRP\n",
      "\n",
      "'m\tVBP\n",
      "\n",
      "inspired\tVBN\n",
      "\n",
      "by\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "mood\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "people\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "''\t''\n",
      "\n",
      "\n",
      "\n",
      "Under\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "laws\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "land\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "remains\tVBZ\n",
      "\n",
      "an\tDT\n",
      "\n",
      "illegal\tJJ\n",
      "\n",
      "organization\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "and\tCC\n",
      "\n",
      "its\tPRP$\n",
      "\n",
      "headquarters\tNNS\n",
      "\n",
      "are\tVBP\n",
      "\n",
      "still\tRB\n",
      "\n",
      "in\tIN\n",
      "\n",
      "Lusaka\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "Zambia\tNNP\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "But\tCC\n",
      "\n",
      "the\tDT\n",
      "\n",
      "unconditional\tJJ\n",
      "\n",
      "release\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "seven\tCD\n",
      "\n",
      "leaders\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "who\tWP\n",
      "\n",
      "once\tRB\n",
      "\n",
      "formed\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "intellectual\tJJ\n",
      "\n",
      "and\tCC\n",
      "\n",
      "organizational\tJJ\n",
      "\n",
      "core\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "is\tVBZ\n",
      "\n",
      "a\tDT\n",
      "\n",
      "de\tFW\n",
      "\n",
      "facto\tFW\n",
      "\n",
      "unbanning\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "movement\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "the\tDT\n",
      "\n",
      "rebirth\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "its\tPRP$\n",
      "\n",
      "internal\tJJ\n",
      "\n",
      "wing\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "``\t``\n",
      "\n",
      "The\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      "can\tMD\n",
      "\n",
      "never\tRB\n",
      "\n",
      "put\tVB\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "back\tNN\n",
      "\n",
      "into\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "bottle\tNN\n",
      "\n",
      "again\tRB\n",
      "\n",
      ",\t,\n",
      "\n",
      "''\t''\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "Cassim\tNNP\n",
      "\n",
      "Saloojee\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "a\tDT\n",
      "\n",
      "veteran\tJJ\n",
      "\n",
      "anti-apartheid\tNN\n",
      "\n",
      "activist\tNN\n",
      "\n",
      "on\tIN\n",
      "\n",
      "hand\tNN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "welcome\tVB\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Sisulu\tNNP\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "``\t``\n",
      "\n",
      "Things\tNNS\n",
      "\n",
      "have\tVBP\n",
      "\n",
      "gone\tVBN\n",
      "\n",
      "too\tRB\n",
      "\n",
      "far\tRB\n",
      "\n",
      "for\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "stop\tVB\n",
      "\n",
      "them\tPRP\n",
      "\n",
      "now\tRB\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "There\tEX\n",
      "\n",
      "'s\tVBZ\n",
      "\n",
      "no\tDT\n",
      "\n",
      "turning\tVBG\n",
      "\n",
      "back\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "''\t''\n",
      "\n",
      "\n",
      "\n",
      "There\tEX\n",
      "\n",
      "was\tVBD\n",
      "\n",
      "certainly\tRB\n",
      "\n",
      "no\tRB\n",
      "\n",
      "stopping\tVBG\n",
      "\n",
      "the\tDT\n",
      "\n",
      "tide\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "emotion\tNN\n",
      "\n",
      "last\tJJ\n",
      "\n",
      "night\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "when\tWRB\n",
      "\n",
      "hundreds\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "people\tNNS\n",
      "\n",
      "jammed\tVBD\n",
      "\n",
      "into\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "Holy\tNNP\n",
      "\n",
      "Cross\tNNP\n",
      "\n",
      "Anglican\tNNP\n",
      "\n",
      "Church\tNNP\n",
      "\n",
      "in\tIN\n",
      "\n",
      "Soweto\tNNP\n",
      "\n",
      "for\tIN\n",
      "\n",
      "what\tWP\n",
      "\n",
      "became\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "first\tJJ\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "rally\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "country\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "30\tCD\n",
      "\n",
      "years\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Deafening\tVBG\n",
      "\n",
      "chants\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "``\t``\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "''\t''\n",
      "\n",
      "and\tCC\n",
      "\n",
      "``\t``\n",
      "\n",
      "Umkhonto\tNNP\n",
      "\n",
      "we\tPRP\n",
      "\n",
      "Sizwe\tNNP\n",
      "\n",
      "''\t''\n",
      "\n",
      "shook\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "church\tNN\n",
      "\n",
      "as\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "seven\tCD\n",
      "\n",
      "aging\tVBG\n",
      "\n",
      "men\tNNS\n",
      "\n",
      "vowed\tVBD\n",
      "\n",
      "that\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "would\tMD\n",
      "\n",
      "continue\tVB\n",
      "\n",
      "its\tPRP$\n",
      "\n",
      "fight\tNN\n",
      "\n",
      "against\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "the\tDT\n",
      "\n",
      "policies\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "racial\tJJ\n",
      "\n",
      "segregation\tNN\n",
      "\n",
      "on\tIN\n",
      "\n",
      "all\tDT\n",
      "\n",
      "fronts\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "including\tVBG\n",
      "\n",
      "the\tDT\n",
      "\n",
      "armed\tVBN\n",
      "\n",
      "struggle\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "And\tCC\n",
      "\n",
      "they\tPRP\n",
      "\n",
      "called\tVBD\n",
      "\n",
      "on\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "release\tVB\n",
      "\n",
      "Nelson\tNNP\n",
      "\n",
      "Mandela\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "leading\tVBG\n",
      "\n",
      "figure\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "who\tWP\n",
      "\n",
      "was\tVBD\n",
      "\n",
      "jailed\tVBN\n",
      "\n",
      "with\tIN\n",
      "\n",
      "them\tPRP\n",
      "\n",
      "and\tCC\n",
      "\n",
      "remains\tVBZ\n",
      "\n",
      "in\tIN\n",
      "\n",
      "prison\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Without\tIN\n",
      "\n",
      "him\tPRP\n",
      "\n",
      ",\t,\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Sisulu\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "freeing\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "others\tNNS\n",
      "\n",
      "``\t``\n",
      "\n",
      "is\tVBZ\n",
      "\n",
      "only\tRB\n",
      "\n",
      "a\tDT\n",
      "\n",
      "half-measure\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "''\t''\n",
      "\n",
      "\n",
      "\n",
      "President\tNNP\n",
      "\n",
      "F.W.\tNNP\n",
      "\n",
      "de\tNNP\n",
      "\n",
      "Klerk\tNNP\n",
      "\n",
      "released\tVBD\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "men\tNNS\n",
      "\n",
      "--\t:\n",
      "\n",
      "along\tIN\n",
      "\n",
      "with\tIN\n",
      "\n",
      "one\tCD\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "founding\tVBG\n",
      "\n",
      "members\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "Pan\tNNP\n",
      "\n",
      "Africanist\tNNP\n",
      "\n",
      "Congress\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "a\tDT\n",
      "\n",
      "rival\tJJ\n",
      "\n",
      "liberation\tNN\n",
      "\n",
      "group\tNN\n",
      "\n",
      "--\t:\n",
      "\n",
      "as\tIN\n",
      "\n",
      "part\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "his\tPRP$\n",
      "\n",
      "efforts\tNNS\n",
      "\n",
      "to\tTO\n",
      "\n",
      "create\tVB\n",
      "\n",
      "a\tDT\n",
      "\n",
      "climate\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "trust\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "peace\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "which\tWDT\n",
      "\n",
      "his\tPRP$\n",
      "\n",
      "government\tNN\n",
      "\n",
      "can\tMD\n",
      "\n",
      "begin\tVB\n",
      "\n",
      "negotiations\tNNS\n",
      "\n",
      "with\tIN\n",
      "\n",
      "black\tJJ\n",
      "\n",
      "leaders\tNNS\n",
      "\n",
      "over\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "new\tJJ\n",
      "\n",
      "constitution\tNN\n",
      "\n",
      "aimed\tVBN\n",
      "\n",
      "at\tIN\n",
      "\n",
      "giving\tVBG\n",
      "\n",
      "blacks\tNNS\n",
      "\n",
      "a\tDT\n",
      "\n",
      "voice\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "national\tJJ\n",
      "\n",
      "government\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "But\tCC\n",
      "\n",
      "Pretoria\tNNP\n",
      "\n",
      "may\tMD\n",
      "\n",
      "instead\tRB\n",
      "\n",
      "be\tVB\n",
      "\n",
      "creating\tVBG\n",
      "\n",
      "a\tDT\n",
      "\n",
      "climate\tNN\n",
      "\n",
      "for\tIN\n",
      "\n",
      "more\tJJR\n",
      "\n",
      "turmoil\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "uncertainty\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "this\tDT\n",
      "\n",
      "racially\tRB\n",
      "\n",
      "divided\tVBN\n",
      "\n",
      "country\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "As\tIN\n",
      "\n",
      "other\tJJ\n",
      "\n",
      "repressive\tJJ\n",
      "\n",
      "governments\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "particularly\tRB\n",
      "\n",
      "Poland\tNNP\n",
      "\n",
      "and\tCC\n",
      "\n",
      "the\tDT\n",
      "\n",
      "Soviet\tNNP\n",
      "\n",
      "Union\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "have\tVBP\n",
      "\n",
      "recently\tRB\n",
      "\n",
      "discovered\tVBN\n",
      "\n",
      ",\t,\n",
      "\n",
      "initial\tJJ\n",
      "\n",
      "steps\tNNS\n",
      "\n",
      "to\tTO\n",
      "\n",
      "open\tVB\n",
      "\n",
      "up\tRP\n",
      "\n",
      "society\tNN\n",
      "\n",
      "can\tMD\n",
      "\n",
      "create\tVB\n",
      "\n",
      "a\tDT\n",
      "\n",
      "momentum\tNN\n",
      "\n",
      "for\tIN\n",
      "\n",
      "radical\tJJ\n",
      "\n",
      "change\tNN\n",
      "\n",
      "that\tWDT\n",
      "\n",
      "becomes\tVBZ\n",
      "\n",
      "difficult\tJJ\n",
      "\n",
      ",\t,\n",
      "\n",
      "if\tIN\n",
      "\n",
      "not\tRB\n",
      "\n",
      "impossible\tJJ\n",
      "\n",
      ",\t,\n",
      "\n",
      "to\tTO\n",
      "\n",
      "control\tVB\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "As\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "days\tNNS\n",
      "\n",
      "go\tVBP\n",
      "\n",
      "by\tIN\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "South\tJJ\n",
      "\n",
      "African\tJJ\n",
      "\n",
      "government\tNN\n",
      "\n",
      "will\tMD\n",
      "\n",
      "be\tVB\n",
      "\n",
      "ever\tRB\n",
      "\n",
      "more\tRBR\n",
      "\n",
      "hard\tRB\n",
      "\n",
      "pressed\tVBN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "justify\tVB\n",
      "\n",
      "the\tDT\n",
      "\n",
      "continued\tVBN\n",
      "\n",
      "imprisonment\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Mandela\tNNP\n",
      "\n",
      "as\tRB\n",
      "\n",
      "well\tRB\n",
      "\n",
      "as\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "continued\tVBN\n",
      "\n",
      "banning\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "and\tCC\n",
      "\n",
      "enforcement\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "state\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "emergency\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "If\tIN\n",
      "\n",
      "it\tPRP\n",
      "\n",
      "does\tVBZ\n",
      "\n",
      "n't\tRB\n",
      "\n",
      "yield\tVB\n",
      "\n",
      "on\tIN\n",
      "\n",
      "these\tDT\n",
      "\n",
      "matters\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "and\tCC\n",
      "\n",
      "eventually\tRB\n",
      "\n",
      "begin\tVB\n",
      "\n",
      "talking\tVBG\n",
      "\n",
      "directly\tRB\n",
      "\n",
      "to\tTO\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "expectations\tNNS\n",
      "\n",
      "and\tCC\n",
      "\n",
      "promise\tNN\n",
      "\n",
      "raised\tVBN\n",
      "\n",
      "by\tIN\n",
      "\n",
      "yesterday\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "releases\tNNS\n",
      "\n",
      "will\tMD\n",
      "\n",
      "turn\tVB\n",
      "\n",
      "to\tTO\n",
      "\n",
      "disillusionment\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "unrest\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "If\tIN\n",
      "\n",
      "it\tPRP\n",
      "\n",
      "does\tVBZ\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "large\tJJ\n",
      "\n",
      "number\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "right-wing\tJJ\n",
      "\n",
      "whites\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "who\tWP\n",
      "\n",
      "oppose\tVBP\n",
      "\n",
      "any\tDT\n",
      "\n",
      "concessions\tNNS\n",
      "\n",
      "to\tTO\n",
      "\n",
      "the\tDT\n",
      "\n",
      "black\tJJ\n",
      "\n",
      "majority\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "will\tMD\n",
      "\n",
      "step\tVB\n",
      "\n",
      "up\tRP\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "agitation\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "threats\tNNS\n",
      "\n",
      "to\tTO\n",
      "\n",
      "take\tVB\n",
      "\n",
      "matters\tNNS\n",
      "\n",
      "into\tIN\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "own\tJJ\n",
      "\n",
      "hands\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "newly\tRB\n",
      "\n",
      "released\tVBN\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "leaders\tNNS\n",
      "\n",
      "also\tRB\n",
      "\n",
      "will\tMD\n",
      "\n",
      "be\tVB\n",
      "\n",
      "under\tIN\n",
      "\n",
      "enormous\tJJ\n",
      "\n",
      "pressure\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      "is\tVBZ\n",
      "\n",
      "watching\tVBG\n",
      "\n",
      "closely\tRB\n",
      "\n",
      "to\tTO\n",
      "\n",
      "see\tVB\n",
      "\n",
      "if\tIN\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "presence\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "townships\tNNS\n",
      "\n",
      "leads\tVBZ\n",
      "\n",
      "to\tTO\n",
      "\n",
      "increased\tVBN\n",
      "\n",
      "anti-government\tNN\n",
      "\n",
      "protests\tNNS\n",
      "\n",
      "and\tCC\n",
      "\n",
      "violence\tNN\n",
      "\n",
      ";\t:\n",
      "\n",
      "if\tIN\n",
      "\n",
      "it\tPRP\n",
      "\n",
      "does\tVBZ\n",
      "\n",
      ",\t,\n",
      "\n",
      "Pretoria\tNNP\n",
      "\n",
      "will\tMD\n",
      "\n",
      "use\tVB\n",
      "\n",
      "this\tDT\n",
      "\n",
      "as\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "reason\tNN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "keep\tVB\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Mandela\tNNP\n",
      "\n",
      "behind\tIN\n",
      "\n",
      "bars\tNNS\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Pretoria\tNNP\n",
      "\n",
      "has\tVBZ\n",
      "\n",
      "n't\tRB\n",
      "\n",
      "forgotten\tVBN\n",
      "\n",
      "why\tWRB\n",
      "\n",
      "they\tPRP\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "all\tDT\n",
      "\n",
      "sentenced\tVBN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "life\tNN\n",
      "\n",
      "imprisonment\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "first\tJJ\n",
      "\n",
      "place\tNN\n",
      "\n",
      ":\t:\n",
      "\n",
      "for\tIN\n",
      "\n",
      "sabotage\tNN\n",
      "\n",
      "and\tCC\n",
      "\n",
      "conspiracy\tNN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "overthrow\tVB\n",
      "\n",
      "the\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "In\tIN\n",
      "\n",
      "addition\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      "is\tVBZ\n",
      "\n",
      "figuring\tVBG\n",
      "\n",
      "that\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "releases\tNNS\n",
      "\n",
      "could\tMD\n",
      "\n",
      "create\tVB\n",
      "\n",
      "a\tDT\n",
      "\n",
      "split\tNN\n",
      "\n",
      "between\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "internal\tJJ\n",
      "\n",
      "and\tCC\n",
      "\n",
      "external\tJJ\n",
      "\n",
      "wings\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "and\tCC\n",
      "\n",
      "between\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "newly\tRB\n",
      "\n",
      "freed\tVBN\n",
      "\n",
      "leaders\tNNS\n",
      "\n",
      "and\tCC\n",
      "\n",
      "those\tDT\n",
      "\n",
      "activists\tNNS\n",
      "\n",
      "who\tWP\n",
      "\n",
      "have\tVBP\n",
      "\n",
      "emerged\tVBN\n",
      "\n",
      "as\tIN\n",
      "\n",
      "leaders\tNNS\n",
      "\n",
      "inside\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "country\tNN\n",
      "\n",
      "during\tIN\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "imprisonment\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "In\tIN\n",
      "\n",
      "order\tNN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "head\tVB\n",
      "\n",
      "off\tRP\n",
      "\n",
      "any\tDT\n",
      "\n",
      "divisions\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "Mr.\tNNP\n",
      "\n",
      "Mandela\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "in\tIN\n",
      "\n",
      "a\tDT\n",
      "\n",
      "meeting\tNN\n",
      "\n",
      "with\tIN\n",
      "\n",
      "his\tPRP$\n",
      "\n",
      "colleagues\tNNS\n",
      "\n",
      "before\tIN\n",
      "\n",
      "they\tPRP\n",
      "\n",
      "were\tVBD\n",
      "\n",
      "released\tVBN\n",
      "\n",
      ",\t,\n",
      "\n",
      "instructed\tVBD\n",
      "\n",
      "them\tPRP\n",
      "\n",
      "to\tTO\n",
      "\n",
      "report\tVB\n",
      "\n",
      "to\tTO\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      "headquarters\tNN\n",
      "\n",
      "in\tIN\n",
      "\n",
      "Lusaka\tNNP\n",
      "\n",
      "as\tRB\n",
      "\n",
      "soon\tRB\n",
      "\n",
      "as\tIN\n",
      "\n",
      "possible\tJJ\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "The\tDT\n",
      "\n",
      "men\tNNS\n",
      "\n",
      "also\tRB\n",
      "\n",
      "will\tMD\n",
      "\n",
      "be\tVB\n",
      "\n",
      "faced\tVBN\n",
      "\n",
      "with\tIN\n",
      "\n",
      "bridging\tVBG\n",
      "\n",
      "the\tDT\n",
      "\n",
      "generation\tNN\n",
      "\n",
      "gap\tNN\n",
      "\n",
      "between\tIN\n",
      "\n",
      "themselves\tPRP\n",
      "\n",
      "and\tCC\n",
      "\n",
      "the\tDT\n",
      "\n",
      "country\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "many\tJJ\n",
      "\n",
      "militant\tJJ\n",
      "\n",
      "black\tJJ\n",
      "\n",
      "youths\tNNS\n",
      "\n",
      ",\t,\n",
      "\n",
      "the\tDT\n",
      "\n",
      "so-called\tJJ\n",
      "\n",
      "young\tJJ\n",
      "\n",
      "lions\tNNS\n",
      "\n",
      "who\tWP\n",
      "\n",
      "are\tVBP\n",
      "\n",
      "anxious\tJJ\n",
      "\n",
      "to\tTO\n",
      "\n",
      "see\tVB\n",
      "\n",
      "the\tDT\n",
      "\n",
      "old\tJJ\n",
      "\n",
      "lions\tNNS\n",
      "\n",
      "in\tIN\n",
      "\n",
      "action\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "Says\tVBZ\n",
      "\n",
      "Peter\tNNP\n",
      "\n",
      "Mokaba\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "president\tNN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "South\tNNP\n",
      "\n",
      "African\tNNP\n",
      "\n",
      "Youth\tNNP\n",
      "\n",
      "Congress\tNNP\n",
      "\n",
      ":\t:\n",
      "\n",
      "``\t``\n",
      "\n",
      "We\tPRP\n",
      "\n",
      "will\tMD\n",
      "\n",
      "be\tVB\n",
      "\n",
      "expecting\tVBG\n",
      "\n",
      "them\tPRP\n",
      "\n",
      "to\tTO\n",
      "\n",
      "act\tVB\n",
      "\n",
      "like\tIN\n",
      "\n",
      "leaders\tNNS\n",
      "\n",
      "of\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "ANC\tNNP\n",
      "\n",
      ".\t.\n",
      "\n",
      "''\t''\n",
      "\n",
      "\n",
      "\n",
      "They\tPRP\n",
      "\n",
      "never\tRB\n",
      "\n",
      "considered\tVBD\n",
      "\n",
      "themselves\tPRP\n",
      "\n",
      "to\tTO\n",
      "\n",
      "be\tVB\n",
      "\n",
      "anything\tNN\n",
      "\n",
      "else\tRB\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "At\tIN\n",
      "\n",
      "last\tJJ\n",
      "\n",
      "night\tNN\n",
      "\n",
      "'s\tPOS\n",
      "\n",
      "rally\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "they\tPRP\n",
      "\n",
      "called\tVBD\n",
      "\n",
      "on\tIN\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "followers\tNNS\n",
      "\n",
      "to\tTO\n",
      "\n",
      "be\tVB\n",
      "\n",
      "firm\tJJ\n",
      "\n",
      ",\t,\n",
      "\n",
      "yet\tRB\n",
      "\n",
      "disciplined\tVBN\n",
      "\n",
      ",\t,\n",
      "\n",
      "in\tIN\n",
      "\n",
      "their\tPRP$\n",
      "\n",
      "opposition\tNN\n",
      "\n",
      "to\tTO\n",
      "\n",
      "apartheid\tNN\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "``\t``\n",
      "\n",
      "We\tPRP\n",
      "\n",
      "emphasize\tVBP\n",
      "\n",
      "discipline\tNN\n",
      "\n",
      "because\tIN\n",
      "\n",
      "we\tPRP\n",
      "\n",
      "know\tVBP\n",
      "\n",
      "that\tIN\n",
      "\n",
      "the\tDT\n",
      "\n",
      "government\tNN\n",
      "\n",
      "is\tVBZ\n",
      "\n",
      "very\tRB\n",
      "\n",
      ",\t,\n",
      "\n",
      "very\tRB\n",
      "\n",
      "sensitive\tJJ\n",
      "\n",
      ",\t,\n",
      "\n",
      "''\t''\n",
      "\n",
      "said\tVBD\n",
      "\n",
      "Andrew\tNNP\n",
      "\n",
      "Mlangeni\tNNP\n",
      "\n",
      ",\t,\n",
      "\n",
      "another\tDT\n",
      "\n",
      "early\tJJ\n",
      "\n",
      "Umkhonto\tNNP\n",
      "\n",
      "leader\tNN\n",
      "\n",
      "who\tWP\n",
      "\n",
      "is\tVBZ\n",
      "\n",
      "now\tRB\n",
      "\n",
      "63\tCD\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n",
      "``\t``\n",
      "\n",
      "We\tPRP\n",
      "\n",
      "want\tVBP\n",
      "\n",
      "to\tTO\n",
      "\n",
      "see\tVB\n",
      "\n",
      "Nelson\tNNP\n",
      "\n",
      "Mandela\tNNP\n",
      "\n",
      "and\tCC\n",
      "\n",
      "all\tPDT\n",
      "\n",
      "our\tPRP$\n",
      "\n",
      "comrades\tNNS\n",
      "\n",
      "out\tIN\n",
      "\n",
      "of\tIN\n",
      "\n",
      "prison\tNN\n",
      "\n",
      ",\t,\n",
      "\n",
      "and\tCC\n",
      "\n",
      "if\tIN\n",
      "\n",
      "we\tPRP\n",
      "\n",
      "are\tVBP\n",
      "\n",
      "n't\tRB\n",
      "\n",
      "disciplined\tVBN\n",
      "\n",
      "we\tPRP\n",
      "\n",
      "may\tMD\n",
      "\n",
      "not\tRB\n",
      "\n",
      "see\tVB\n",
      "\n",
      "them\tPRP\n",
      "\n",
      "here\tRB\n",
      "\n",
      "with\tIN\n",
      "\n",
      "us\tPRP\n",
      "\n",
      ".\t.\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with open(\"WSJ_24.pos\", 'r') as f:\n",
    "    test_set = f.readlines()\n",
    "\n",
    "for i in test_set:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['The', 'economy', \"'s\", 'temperature', 'will', 'be', 'taken', 'from', 'several', '--unknow--']\n"
     ]
    }
   ],
   "source": [
    "_, test_set_without_tag = preprocess(vocabulary_with_index, \"test.words\")\n",
    "print(test_set_without_tag[0:10])"
   ]
  },
  {
   "source": [
    "## Build MFC"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "### Create 3 dictionaries:\n",
    " - transition dictionary: maps (prev_tag, tag) to the number of times it has appeared\n",
    " - emission_dictionary: maps (tag, word) to the number of times it happended\n",
    " - tag_dictionary: maps (tag) to the number of times it has occured "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "### pass into an training set and vocabulary then return 3 dictionaries above\n",
    "\n",
    "def create_dictionaries(training_set, vocabulary):\n",
    "    transition_dictionary = defaultdict(int)\n",
    "    emission_dictionary = defaultdict(int)\n",
    "    tag_dictionary = defaultdict(int)\n",
    "    \n",
    "    # define tag for the begining state (begining tag)\n",
    "    prev_tag = '--s--'\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    for word_tag in training_set:\n",
    "        i += 1\n",
    "        \n",
    "        if i % 50000 == 0:\n",
    "            print(f\"word count = {i}\") \n",
    "\n",
    "        if word_tag == '\\n':\n",
    "            prev_tag = '--s--'\n",
    "            tag_dictionary['--s--'] += 1\n",
    "            continue\n",
    "\n",
    "        word, tag = get_word_tag(word_tag, vocabulary)\n",
    "\n",
    "        transition_dictionary[(prev_tag, tag)] += 1\n",
    "\n",
    "        emission_dictionary[(tag, word)] += 1\n",
    "\n",
    "        tag_dictionary[tag] += 1\n",
    "\n",
    "        prev_tag = tag\n",
    "\n",
    "    return transition_dictionary, emission_dictionary, tag_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "word count = 50000\n",
      "word count = 100000\n",
      "word count = 150000\n",
      "word count = 200000\n",
      "word count = 250000\n",
      "word count = 300000\n",
      "word count = 350000\n",
      "word count = 400000\n",
      "word count = 450000\n",
      "word count = 500000\n",
      "word count = 550000\n",
      "word count = 600000\n",
      "word count = 650000\n",
      "word count = 700000\n",
      "word count = 750000\n",
      "word count = 800000\n",
      "word count = 850000\n",
      "word count = 900000\n",
      "word count = 950000\n"
     ]
    }
   ],
   "source": [
    "transition_dictionary, emission_dictionary, tag_dictionary = create_dictionaries(training_set, vocabulary_with_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "amount of states:  46\nlist of states: \n['#', '$', \"''\", '(', ')', ',', '--s--', '.', ':', 'CC', 'CD', 'DT', 'EX', 'FW', 'IN', 'JJ', 'JJR', 'JJS', 'LS', 'MD', 'NN', 'NNP', 'NNPS', 'NNS', 'PDT', 'POS', 'PRP', 'PRP$', 'RB', 'RBR', 'RBS', 'RP', 'SYM', 'TO', 'UH', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'WDT', 'WP', 'WP$', 'WRB', '``']\n"
     ]
    }
   ],
   "source": [
    "states = sorted(tag_dictionary.keys())\n",
    "print('amount of states: ', len(states))\n",
    "print('list of states: ')\n",
    "print(states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(('--s--', 'IN'), 5050)\n(('IN', 'DT'), 32364)\n(('DT', 'NNP'), 9044)\n(('NNP', 'CD'), 1752)\n(('CD', 'NN'), 7377)\n(('NN', 'IN'), 32885)\n(('IN', '``'), 546)\n(('``', 'DT'), 1014)\n(('DT', 'NN'), 38873)\n(('NN', \"''\"), 686)\n"
     ]
    }
   ],
   "source": [
    "for ex in list(transition_dictionary.items())[:10]:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(('IN', 'In'), 1735)\n(('DT', 'an'), 3142)\n(('NNP', 'Oct.'), 317)\n(('CD', '19'), 100)\n(('NN', 'review'), 36)\n(('IN', 'of'), 22925)\n(('``', '``'), 6967)\n(('DT', 'The'), 6795)\n(('NN', 'Misanthrope'), 3)\n((\"''\", \"''\"), 6787)\n"
     ]
    }
   ],
   "source": [
    "for ex in list(emission_dictionary.items())[:10]:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "('NN', 'review') 36\n('VB', 'review') 21\n('VBP', 'review') 1\n"
     ]
    }
   ],
   "source": [
    "for word, count in emission_dictionary.items():\n",
    "    if word[1] == 'review':\n",
    "        print(word, count) "
   ]
  },
  {
   "source": [
    "### Create predict function"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(test_set_without_tag, test_set, emission_dictionary, vocabulary, states):\n",
    "\n",
    "    # define the number of correction word\n",
    "    correct = 0\n",
    "\n",
    "    # define a variable get all words in the training_set\n",
    "    all_words = set(emission_dictionary.keys())\n",
    "\n",
    "    # define a value get number of words in test set\n",
    "    total = len(test_set)\n",
    "\n",
    "    # predict\n",
    "    for word, word_with_tag in zip(test_set_without_tag, test_set):\n",
    "\n",
    "        # get word and tag in the test_set with tag and split it\n",
    "        item = word_with_tag.split()\n",
    "\n",
    "        # get the true label in test_set\n",
    "        if len(item) == 2:\n",
    "            true_label = item[1]\n",
    "        else:\n",
    "            continue\n",
    "        \n",
    "        # define a variable with the value is the time they appear with a tag\n",
    "        count_final = 0\n",
    "        pos_final = ''\n",
    "\n",
    "        #search word in vocabulary\n",
    "        if word in vocabulary:\n",
    "            # predict base on the time it appears with a tag \n",
    "            for pos in states:\n",
    "                key = (pos, word)\n",
    "\n",
    "                if key in emission_dictionary:\n",
    "                    count = emission_dictionary[key]\n",
    "\n",
    "                    if count > count_final:\n",
    "                        count_final = count\n",
    "                        pos_final = pos\n",
    "            if pos_final == true_label:\n",
    "                correct += 1\n",
    "\n",
    "    accuracy = correct / total\n",
    "    return accuracy    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.8888563993099213\n"
     ]
    }
   ],
   "source": [
    "accuracy = predict(test_set_without_tag, test_set, emission_dictionary, vocabulary_with_index, states)\n",
    "print(accuracy)"
   ]
  },
  {
   "source": [
    "## Build HMM for POS (Viterbi_algorithm)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "### Build matrices:\n",
    "- Creating the 'A' transition probabilites matrix\n",
    "- Creating the 'B' emission probabilites matrix"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "#### Create the 'A' transition probabilities matrix\n"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_transition_probabilities_matrix(alpha, tag_dictionary, transition_dictionary):\n",
    "    all_tags = sorted(tag_dictionary.keys())\n",
    "\n",
    "    num_tags = len(all_tags)\n",
    "\n",
    "    A = np.zeros((num_tags, num_tags))\n",
    "\n",
    "    trans_key = set(transition_dictionary.keys())\n",
    "\n",
    "    for i in range(num_tags):\n",
    "        for j in range(num_tags):\n",
    "            count = 0\n",
    "\n",
    "            key = (all_tags[i], all_tags[j])\n",
    "\n",
    "            if key in transition_dictionary:\n",
    "                count = transition_dictionary[key]\n",
    "            \n",
    "            count_prev = tag_dictionary[all_tags[i]]\n",
    "\n",
    "            A[i, j] = (count +alpha)/ (count_prev + alpha * num_tags)\n",
    "    return A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.001\n",
    "\n",
    "A = create_transition_probabilities_matrix(alpha, tag_dictionary, transition_dictionary)\n",
    "\n",
    "A_dataframe = pd.DataFrame(A, index=states, columns = states)\n",
    "A_dataframe.to_csv('transition_probabilities_matrix.csv')"
   ]
  },
  {
   "source": [
    "#### Create the 'B' emission probabilities matrix\n"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_emission_probabilities_matrix(alpha, tag_dictionary, emission_dictionary, vocabulary):\n",
    "    num_tags = len(tag_dictionary)\n",
    "\n",
    "    num_words = len(vocabulary)\n",
    "\n",
    "    all_tags = sorted(tag_dictionary.keys())\n",
    "\n",
    "    B = np.zeros((num_tags, num_words))\n",
    "\n",
    "    emis_keys = set(emission_dictionary.keys())\n",
    "\n",
    "    for i in range(num_tags):\n",
    "        for j in range(num_words):\n",
    "            count = 0\n",
    "\n",
    "            key = (all_tags[i], vocabulary[j])\n",
    "\n",
    "            if key in emission_dictionary:\n",
    "                count = emission_dictionary[key]\n",
    "            \n",
    "            count_tag = tag_dictionary[all_tags[i]]\n",
    "\n",
    "            B[i, j] = (count + alpha) / (count_tag + alpha * num_words)\n",
    "    return B        \n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "B = create_emission_probabilities_matrix(alpha, tag_dictionary, emission_dictionary, list(vocabulary_with_index))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "B_dataframe = pd.DataFrame(B, index=states, columns = list(vocabulary_with_index))\n",
    "B_dataframe.to_csv('emission_probabilities_matrix.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "              725      adroitly     engineers      promoted       synergy\nCD   8.201296e-05  2.732854e-08  2.732854e-08  2.732854e-08  2.732854e-08\nNN   7.521128e-09  7.521128e-09  7.521128e-09  7.521128e-09  2.257091e-05\nNNS  1.670013e-08  1.670013e-08  4.676203e-04  1.670013e-08  1.670013e-08\nVB   3.779036e-08  3.779036e-08  3.779036e-08  3.779036e-08  3.779036e-08\nRB   3.226454e-08  6.456135e-05  3.226454e-08  3.226454e-08  3.226454e-08\nRP   3.723317e-07  3.723317e-07  3.723317e-07  3.723317e-07  3.723317e-07\n"
     ]
    }
   ],
   "source": [
    "cidx  = ['725','adroitly','engineers', 'promoted', 'synergy']\n",
    "\n",
    "cols = [vocabulary_with_index\n",
    "[a] for a in cidx]\n",
    "\n",
    "rvals =['CD','NN','NNS', 'VB','RB','RP']\n",
    "\n",
    "rows = [states.index(a) for a in rvals]\n",
    "\n",
    "B_sub = pd.DataFrame(B[np.ix_(rows,cols)], index=rvals, columns = cidx )\n",
    "print(B_sub)"
   ]
  },
  {
   "source": [
    "## Viterbi algorithm"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "### initialization"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize(states, tag_dictionary, A, B, corpus, vocab):\n",
    "    num_tags = len(tag_dictionary)\n",
    "\n",
    "    best_probs = np.zeros((num_tags, len(corpus)))\n",
    "\n",
    "    best_paths = np.zeros((num_tags, len(corpus)))\n",
    "\n",
    "    s_idx = states.index(\"--s--\")\n",
    "\n",
    "    for i in range(num_tags):\n",
    "        if A[s_idx, i] == 0:\n",
    "            best_props[i, 0] = -float('inf')\n",
    "        else:\n",
    "            best_probs[i,0] = np.log(A[s_idx, i]) + np.log(B[i, vocab[corpus[0]]])\n",
    "\n",
    "    return best_probs, best_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_probs, best_paths = initialize(states, tag_dictionary, A, B, test_set_without_tag, vocabulary_with_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "best_probs[0,0]: -22.6098\nbest_paths[2,3]: 0.0000\n"
     ]
    }
   ],
   "source": [
    "print(f\"best_probs[0,0]: {best_probs[0,0]:.4f}\") \n",
    "print(f\"best_paths[2,3]: {best_paths[2,3]:.4f}\")"
   ]
  },
  {
   "source": [
    "## Viterbi foward "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viterbi_fowrad(A, B, test_cropus, best_probs, best_paths, vocab):\n",
    "    num_tags = best_probs.shape[0]\n",
    "\n",
    "    for i in range(1, len(test_cropus)):\n",
    "\n",
    "        if i % 5000 == 0:\n",
    "            print(\"words processed: \", i)\n",
    "        # for each POS tag that current word can be\n",
    "        for j in range(num_tags):\n",
    "            best_prob_i = -float('inf')\n",
    "\n",
    "            best_path_i = None\n",
    "\n",
    "            # for each pos tag that the previous word can be\n",
    "            for k in range(num_tags):\n",
    "\n",
    "                prob = best_probs[k, i-1] + math.log(A[k, j]) + math.log(B[j, vocab[test_cropus[i]]])\n",
    "\n",
    "                if prob > best_prob_i:\n",
    "                    best_prob_i = prob\n",
    "\n",
    "                    best_path_i = k \n",
    "            best_probs[j, i] = best_prob_i\n",
    "\n",
    "            best_paths[j, i] = best_path_i  \n",
    "\n",
    "    return best_probs, best_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "words processed:  5000\n",
      "words processed:  10000\n",
      "words processed:  15000\n",
      "words processed:  20000\n",
      "words processed:  25000\n",
      "words processed:  30000\n"
     ]
    }
   ],
   "source": [
    "best_probs, best_paths = viterbi_fowrad(A, B, test_set_without_tag, best_probs, best_paths, vocabulary_with_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "best_probs[0,1]:  -24.78215632717346\nbest_paths[0,4]:  0.0\n"
     ]
    }
   ],
   "source": [
    "print(\"best_probs[0,1]: \", best_probs[0,1])\n",
    "print(\"best_paths[0,4]: \", best_paths[2,0])"
   ]
  },
  {
   "source": [
    "## Viterbi backward"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viterbi_backward(best_probs, best_paths, corpus, states):\n",
    "    m = best_paths.shape[1]\n",
    "\n",
    "    z = [None] * m\n",
    "\n",
    "    num_tags = best_probs.shape[0]\n",
    "\n",
    "    best_prob_for_last_word = -float('inf')\n",
    "\n",
    "    predict = [None] * m \n",
    "\n",
    "    for k in range(num_tags):\n",
    "        if best_probs[k, m - 1] > best_prob_for_last_word:\n",
    "\n",
    "            best_prob_for_last_word = best_probs[k, m - 1]\n",
    "\n",
    "            z[m - 1] = k\n",
    "\n",
    "    predict[m - 1] = states[z[m-1]]\n",
    "\n",
    "    for i in range(m-1, -1, -1):\n",
    "        pos_tag_for_word_i = z[i]\n",
    "\n",
    "        z[i - 1] = int(best_paths[pos_tag_for_word_i, i])\n",
    "\n",
    "        predict[i - 1] = states[z[i - 1]]\n",
    "\n",
    "    return predict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "predict result:  The \t DT\ntrue label:  The\tDT\n\npredict result:  most \t RBS\ntrue label:  most\tRBS\n\npredict result:  troublesome \t JJ\ntrue label:  troublesome\tJJ\n\npredict result:  report \t NN\ntrue label:  report\tNN\n\npredict result:  may \t MD\ntrue label:  may\tMD\n\npredict result:  be \t VB\ntrue label:  be\tVB\n\npredict result:  the \t DT\ntrue label:  the\tDT\n\npredict result:  August \t NNP\ntrue label:  August\tNNP\n\npredict result:  merchandise \t NN\ntrue label:  merchandise\tNN\n\npredict result:  trade \t NN\ntrue label:  trade\tNN\n\npredict result:  deficit \t NN\ntrue label:  deficit\tNN\n\npredict result:  due \t JJ\ntrue label:  due\tJJ\n\npredict result:  out \t IN\ntrue label:  out\tIN\n\npredict result:  tomorrow \t NN\ntrue label:  tomorrow\tNN\n\npredict result:  . \t .\ntrue label:  .\t.\n\n"
     ]
    }
   ],
   "source": [
    "predict = viterbi_backward(best_probs, best_paths, test_set_without_tag, states)\n",
    "m = len(predict)\n",
    "for i in range(26, 41):\n",
    "    print('predict result: ',test_set_without_tag[i], '\\t', predict[i])\n",
    "    print('true label: ',test_set[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['Quang']"
      ]
     },
     "metadata": {},
     "execution_count": 220
    }
   ],
   "source": [
    "string = 'Quang'\n",
    "test = string.split()\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_probs, best_paths = viterbi_fowrad(A, B, test, best_probs, best_paths, vocabulary_with_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "predict result:  Quang \t DT\n"
     ]
    }
   ],
   "source": [
    "predict = viterbi_backward(best_probs, best_paths, test, states)\n",
    "m = len(predict)\n",
    "for i in range(0, 1):\n",
    "    print('predict result: ',test[i], '\\t', predict[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "# UNQ_C8 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)\n",
    "# GRADED FUNCTION: compute_accuracy\n",
    "def compute_accuracy(pred, y):\n",
    "    '''\n",
    "    Input: \n",
    "        pred: a list of the predicted parts-of-speech \n",
    "        y: a list of lines where each word is separated by a '\\t' (i.e. word \\t tag)\n",
    "    Output: \n",
    "        \n",
    "    '''\n",
    "    num_correct = 0\n",
    "    total = 0\n",
    "    \n",
    "    # Zip together the prediction and the labels\n",
    "    for prediction, y in zip(pred, y):\n",
    "        ### START CODE HERE (Replace instances of 'None' with your code) ###\n",
    "        # Split the label into the word and the POS tag\n",
    "        word_tag_tuple = y.split()\n",
    "        \n",
    "        # Check that there is actually a word and a tag\n",
    "        # no more and no less than 2 items\n",
    "        if len(word_tag_tuple) != 2:\n",
    "            continue\n",
    "\n",
    "        # store the word and tag separately\n",
    "        word, tag = word_tag_tuple\n",
    "        \n",
    "        # Check if the POS tag label matches the prediction\n",
    "        if prediction == tag: \n",
    "            \n",
    "            # count the number of times that the prediction\n",
    "            # and label match\n",
    "            num_correct += 1\n",
    "            \n",
    "        # keep track of the total number of examples (that have valid labels)\n",
    "        total += 1\n",
    "        \n",
    "        ### END CODE HERE ###\n",
    "    return num_correct / total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Accuracy of the Viterbi algorithm is 0.9529\n"
     ]
    }
   ],
   "source": [
    "print(f\"Accuracy of the Viterbi algorithm is {compute_accuracy(predict, test_set):.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}