Heading_based_bagofwords.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6fda30af",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "69d07a04",
   "metadata": {},
   "outputs": [],
   "source": [
    "data1 = pd.read_csv(\"data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2321135a",
   "metadata": {},
   "outputs": [],
   "source": [
    "data2=data1.dropna(subset=['URL'])\n",
    "data3 = data2.dropna(subset=['Content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "103828d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.parse import urlparse\n",
    "def is_valid_url(URL):\n",
    "    try:\n",
    "        result = urlparse(URL)\n",
    "        return all([result.scheme, result.netloc])\n",
    "    except ValueError:\n",
    "        return False\n",
    "\n",
    "valid_url_mask = data3['URL'].apply(is_valid_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3059da19",
   "metadata": {},
   "outputs": [],
   "source": [
    "data4 = data3[valid_url_mask]\n",
    "data5= data3[valid_url_mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e0ed4696",
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_nan_with_string(df, column_name, replacement_string='abc'):\n",
    "  \n",
    "    df[column_name] = df[column_name].fillna(replacement_string)\n",
    "    return df\n",
    "\n",
    "data6 = replace_nan_with_string(data4, 'Heading')\n",
    "data6= replace_nan_with_string(data4, 'Content')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3a224c3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL</th>\n",
       "      <th>Heading</th>\n",
       "      <th>Content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.rbi.org.in/hindi/Home.aspx</td>\n",
       "      <td>abc</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://cms.rbi.org.in</td>\n",
       "      <td>abc</td>\n",
       "      <td>Never respond to Request money options from un...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.rbi.org.in/scripts/MANI.aspx</td>\n",
       "      <td>Press Release Public URLs for MANI app Radio J...</td>\n",
       "      <td>Android app is available publicly on httpsplay...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https://Rbikehtahai.rbi.org.in</td>\n",
       "      <td>abc</td>\n",
       "      <td>Safe digital banking starts with you Transact ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>https://therbimuseum.rbi.org.in</td>\n",
       "      <td>here Accessibility</td>\n",
       "      <td>Reserve a Tour Manage Reservation Cancel Reser...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>https://fintech.rbi.org.in</td>\n",
       "      <td>Important</td>\n",
       "      <td>India is on the way to becoming Asias top fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_PressRelease...</td>\n",
       "      <td>Date Supersession of Board of Directors of Abh...</td>\n",
       "      <td>In exercise of the powers conferred under Sect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_PressRelease...</td>\n",
       "      <td>Date RBI updates the Alert List of unauthorize...</td>\n",
       "      <td>The Reserve Bank of India RBI has added the fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_SpeechesView...</td>\n",
       "      <td>Date Changing Paradigms in the Financial Lands...</td>\n",
       "      <td>Distinguished guests Good evening First of all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_SpeechesView...</td>\n",
       "      <td>Date Winning in Uncertain Times The Indian Exp...</td>\n",
       "      <td>I am thankful to FICCI and IBA for once again ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/NotificationUse...</td>\n",
       "      <td>Regulatory measures towards consumer credit an...</td>\n",
       "      <td>RBI20232485 DORSTRREC572106001202324 November ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_PressRelease...</td>\n",
       "      <td>Date RBI Bulletin November 2023</td>\n",
       "      <td>Today the Reserve Bank released the November 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>https://rbi.org.in/Scripts/BS_PressReleaseDisp...</td>\n",
       "      <td>Date Action against Bajaj Finance Ltd under Se...</td>\n",
       "      <td>The Reserve Bank of India has today in exercis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_PressRelease...</td>\n",
       "      <td>Date RBI releases Handbook of Statistics on In...</td>\n",
       "      <td>Today the Reserve Bank of India released the e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>https://rbi.org.in/Scripts/BS_SpeechesView.asp...</td>\n",
       "      <td>Date Emerging India A Land of Stability and Op...</td>\n",
       "      <td>I am delighted to participate in this symposiu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/NotificationUse...</td>\n",
       "      <td>Master Direction on Information Technology Gov...</td>\n",
       "      <td>RBI202324107 DoSCOCSITEGSEC73101015202324 Nove...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_SpeechesView...</td>\n",
       "      <td>Date Towards A Greener Cleaner India Inaugural...</td>\n",
       "      <td>Good Morning to you all I am delighted to be h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_SpeechesView...</td>\n",
       "      <td>Date Governor Shri Shaktikanta Dass interactio...</td>\n",
       "      <td>Raghuvir Srinivasan Editor Business Line Recog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_PressRelease...</td>\n",
       "      <td>Date Release of the India Finance Report of th...</td>\n",
       "      <td>The first flagship publication of the Centre f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_SpeechesView...</td>\n",
       "      <td>Date Reflections Challenges in Regulations</td>\n",
       "      <td>Ladies gentlemen and distinguished guests It i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_PressRelease...</td>\n",
       "      <td>Date RBI appoints Shri Manoranjan Mishra as ne...</td>\n",
       "      <td>The Reserve Bank of India RBI has appointed Sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_PressRelease...</td>\n",
       "      <td>Date Withdrawal of 2000 Denomination Banknotes...</td>\n",
       "      <td>The Reserve Bank of India RBI had announced th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_PressRelease...</td>\n",
       "      <td>Date 604th Meeting of Central Board of the Res...</td>\n",
       "      <td>The 604th meeting of the Central Board of Dire...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>https://www.rbi.org.in/Scripts/BS_PressRelease...</td>\n",
       "      <td>Date RBI invites comments on draft Master Dire...</td>\n",
       "      <td>The Reserve Bank of India has today placed on ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>https://opportunities.rbi.org.in/Scripts/resul...</td>\n",
       "      <td>Nov 09 2023 Nov 07 2023 Nov 03 2023 Oct 27 202...</td>\n",
       "      <td>Caution It has come to the notice of Reserve B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>https://rbikehtahai.rbi.org.in/</td>\n",
       "      <td>abc</td>\n",
       "      <td>Safe digital banking starts with you Transact ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>https://paisaboltahai.rbi.org.in/</td>\n",
       "      <td>abc</td>\n",
       "      <td>Currency notes reflect the nations rich and di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>https://rbi.org.in/Scripts/CitiChart.aspx</td>\n",
       "      <td>abc</td>\n",
       "      <td>Disclaimer The stipulated timelines are subjec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>https://opportunities.rbi.org.in/scripts/index...</td>\n",
       "      <td>abc</td>\n",
       "      <td>Caution It has come to the notice of Reserve B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>https://rbi.org.in/Scripts/Righttoinfoact.aspx</td>\n",
       "      <td>Right to Information Act 2005 RBIs Obligation ...</td>\n",
       "      <td>Please visit Disclosure Log section on RTI pag...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  URL  \\\n",
       "0              https://www.rbi.org.in/hindi/Home.aspx   \n",
       "3                              https://cms.rbi.org.in   \n",
       "4            https://www.rbi.org.in/scripts/MANI.aspx   \n",
       "5                      https://Rbikehtahai.rbi.org.in   \n",
       "6                     https://therbimuseum.rbi.org.in   \n",
       "7                          https://fintech.rbi.org.in   \n",
       "10  https://www.rbi.org.in/Scripts/BS_PressRelease...   \n",
       "11  https://www.rbi.org.in/Scripts/BS_PressRelease...   \n",
       "12  https://www.rbi.org.in/Scripts/BS_SpeechesView...   \n",
       "13  https://www.rbi.org.in/Scripts/BS_SpeechesView...   \n",
       "14  https://www.rbi.org.in/Scripts/NotificationUse...   \n",
       "15  https://www.rbi.org.in/Scripts/BS_PressRelease...   \n",
       "16  https://rbi.org.in/Scripts/BS_PressReleaseDisp...   \n",
       "17  https://www.rbi.org.in/Scripts/BS_PressRelease...   \n",
       "18  https://rbi.org.in/Scripts/BS_SpeechesView.asp...   \n",
       "19  https://www.rbi.org.in/Scripts/NotificationUse...   \n",
       "20  https://www.rbi.org.in/Scripts/BS_SpeechesView...   \n",
       "21  https://www.rbi.org.in/Scripts/BS_SpeechesView...   \n",
       "22  https://www.rbi.org.in/Scripts/BS_PressRelease...   \n",
       "23  https://www.rbi.org.in/Scripts/BS_SpeechesView...   \n",
       "24  https://www.rbi.org.in/Scripts/BS_PressRelease...   \n",
       "25  https://www.rbi.org.in/Scripts/BS_PressRelease...   \n",
       "26  https://www.rbi.org.in/Scripts/BS_PressRelease...   \n",
       "27  https://www.rbi.org.in/Scripts/BS_PressRelease...   \n",
       "28  https://opportunities.rbi.org.in/Scripts/resul...   \n",
       "29                    https://rbikehtahai.rbi.org.in/   \n",
       "30                  https://paisaboltahai.rbi.org.in/   \n",
       "31          https://rbi.org.in/Scripts/CitiChart.aspx   \n",
       "34  https://opportunities.rbi.org.in/scripts/index...   \n",
       "36     https://rbi.org.in/Scripts/Righttoinfoact.aspx   \n",
       "\n",
       "                                              Heading  \\\n",
       "0                                                 abc   \n",
       "3                                                 abc   \n",
       "4   Press Release Public URLs for MANI app Radio J...   \n",
       "5                                                 abc   \n",
       "6                                  here Accessibility   \n",
       "7                                           Important   \n",
       "10  Date Supersession of Board of Directors of Abh...   \n",
       "11  Date RBI updates the Alert List of unauthorize...   \n",
       "12  Date Changing Paradigms in the Financial Lands...   \n",
       "13  Date Winning in Uncertain Times The Indian Exp...   \n",
       "14  Regulatory measures towards consumer credit an...   \n",
       "15                    Date RBI Bulletin November 2023   \n",
       "16  Date Action against Bajaj Finance Ltd under Se...   \n",
       "17  Date RBI releases Handbook of Statistics on In...   \n",
       "18  Date Emerging India A Land of Stability and Op...   \n",
       "19  Master Direction on Information Technology Gov...   \n",
       "20  Date Towards A Greener Cleaner India Inaugural...   \n",
       "21  Date Governor Shri Shaktikanta Dass interactio...   \n",
       "22  Date Release of the India Finance Report of th...   \n",
       "23         Date Reflections Challenges in Regulations   \n",
       "24  Date RBI appoints Shri Manoranjan Mishra as ne...   \n",
       "25  Date Withdrawal of 2000 Denomination Banknotes...   \n",
       "26  Date 604th Meeting of Central Board of the Res...   \n",
       "27  Date RBI invites comments on draft Master Dire...   \n",
       "28  Nov 09 2023 Nov 07 2023 Nov 03 2023 Oct 27 202...   \n",
       "29                                                abc   \n",
       "30                                                abc   \n",
       "31                                                abc   \n",
       "34                                                abc   \n",
       "36  Right to Information Act 2005 RBIs Obligation ...   \n",
       "\n",
       "                                              Content  \n",
       "0                                                      \n",
       "3   Never respond to Request money options from un...  \n",
       "4   Android app is available publicly on httpsplay...  \n",
       "5   Safe digital banking starts with you Transact ...  \n",
       "6   Reserve a Tour Manage Reservation Cancel Reser...  \n",
       "7   India is on the way to becoming Asias top fina...  \n",
       "10  In exercise of the powers conferred under Sect...  \n",
       "11  The Reserve Bank of India RBI has added the fo...  \n",
       "12  Distinguished guests Good evening First of all...  \n",
       "13  I am thankful to FICCI and IBA for once again ...  \n",
       "14  RBI20232485 DORSTRREC572106001202324 November ...  \n",
       "15  Today the Reserve Bank released the November 2...  \n",
       "16  The Reserve Bank of India has today in exercis...  \n",
       "17  Today the Reserve Bank of India released the e...  \n",
       "18  I am delighted to participate in this symposiu...  \n",
       "19  RBI202324107 DoSCOCSITEGSEC73101015202324 Nove...  \n",
       "20  Good Morning to you all I am delighted to be h...  \n",
       "21  Raghuvir Srinivasan Editor Business Line Recog...  \n",
       "22  The first flagship publication of the Centre f...  \n",
       "23  Ladies gentlemen and distinguished guests It i...  \n",
       "24  The Reserve Bank of India RBI has appointed Sh...  \n",
       "25  The Reserve Bank of India RBI had announced th...  \n",
       "26  The 604th meeting of the Central Board of Dire...  \n",
       "27  The Reserve Bank of India has today placed on ...  \n",
       "28  Caution It has come to the notice of Reserve B...  \n",
       "29  Safe digital banking starts with you Transact ...  \n",
       "30  Currency notes reflect the nations rich and di...  \n",
       "31  Disclaimer The stipulated timelines are subjec...  \n",
       "34  Caution It has come to the notice of Reserve B...  \n",
       "36  Please visit Disclosure Log section on RTI pag...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "def clean_text(text):\n",
    "  \n",
    "    text = BeautifulSoup(text, 'html.parser').get_text()\n",
    "   \n",
    "    text = re.sub(r'[^a-zA-Z0-9\\s]', '', text)\n",
    "    text = ' '.join(text.split())\n",
    "\n",
    "    return text\n",
    "\n",
    "data6[['Content', 'Heading']] = data6[['Content', 'Heading']].applymap(clean_text)\n",
    "\n",
    "data6.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "22a9b9c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL</th>\n",
       "      <th>RowNumber</th>\n",
       "      <th>MergedContent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.rbi.org.in/hindi/Home.aspx</td>\n",
       "      <td>0</td>\n",
       "      <td>abc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://cms.rbi.org.in</td>\n",
       "      <td>3</td>\n",
       "      <td>abc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.rbi.org.in/scripts/MANI.aspx</td>\n",
       "      <td>4</td>\n",
       "      <td>Press Release Public URLs for MANI app Radio J...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https://Rbikehtahai.rbi.org.in</td>\n",
       "      <td>5</td>\n",
       "      <td>abc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>https://therbimuseum.rbi.org.in</td>\n",
       "      <td>6</td>\n",
       "      <td>here Accessibility</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        URL  RowNumber  \\\n",
       "0    https://www.rbi.org.in/hindi/Home.aspx          0   \n",
       "3                    https://cms.rbi.org.in          3   \n",
       "4  https://www.rbi.org.in/scripts/MANI.aspx          4   \n",
       "5            https://Rbikehtahai.rbi.org.in          5   \n",
       "6           https://therbimuseum.rbi.org.in          6   \n",
       "\n",
       "                                       MergedContent  \n",
       "0                                                abc  \n",
       "3                                                abc  \n",
       "4  Press Release Public URLs for MANI app Radio J...  \n",
       "5                                                abc  \n",
       "6                                 here Accessibility  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df = pd.DataFrame({\n",
    "    'URL': data6['URL'],\n",
    "    'RowNumber': data6.index,\n",
    "    'MergedContent': data6['Heading']\n",
    "})\n",
    "result_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a700c81a",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df['MergedContent'] = result_df['MergedContent'].apply(lambda x: x.lower() if isinstance(x, str) else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c15299c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     RowNumber                                            Content\n",
      "0            0                                                abc\n",
      "3            3                                                abc\n",
      "4            4  press release public urls for mani app radio j...\n",
      "5            5                                                abc\n",
      "6            6                                 here accessibility\n",
      "..         ...                                                ...\n",
      "501        501                                       date india75\n",
      "502        502  survey on foreign liabilities and assets fla o...\n",
      "503        503      coordinated portfolio investment survey india\n",
      "504        504  biennial survey on foreign collaboration in in...\n",
      "505        505  annual survey on computer software information...\n",
      "\n",
      "[420 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "vector_df = pd.DataFrame({\n",
    "    'RowNumber': result_df.index,\n",
    "    'Content':result_df['MergedContent']\n",
    "})\n",
    "print(vector_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "49faecdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1dc9ab6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv= CountVectorizer(max_features=5000,stop_words=\"english\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7e7477ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "36adeda0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.porter import PorterStemmer\n",
    "ps= PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6299b82b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def stem(text):\n",
    "    y=[]\n",
    "    for i in text.split():\n",
    "    \n",
    "        y.append(ps.stem(i))\n",
    "    return \" \".join(y)\n",
    "        \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ce9f7fab",
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_df['Content']=vector_df['Content'].apply(stem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "638514aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_documents = vector_df['Content'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "16d24e6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorized_data = cv.fit_transform(raw_documents).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2380b848",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "\n",
    "def find_similar_documents(input_text, cv, vectorized_data, result_df):\n",
    "  \n",
    "    input_vector = cv.transform([input_text]).toarray()\n",
    "\n",
    "    similarities = cosine_similarity(input_vector, vectorized_data)\n",
    "    top_indices = np.argsort(similarities[0])[::-1][:5]\n",
    "    similar_documents = result_df.iloc[top_indices]\n",
    "\n",
    "    return similar_documents\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2264b73a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                       URL  \\\n",
      "124                     https://www.rbi.org.in/Scripts/FAQView.aspx?Id=134   \n",
      "344                https://www.rbi.org.in/scripts/FS_FAQs.aspx?Id=134&fn=6   \n",
      "106  https://www.rbi.org.in/Scripts/BS_PressReleaseDisplay.aspx?prid=49628   \n",
      "110  https://www.rbi.org.in/Scripts/BS_PressReleaseDisplay.aspx?prid=49583   \n",
      "103  https://www.rbi.org.in/Scripts/BS_PressReleaseDisplay.aspx?prid=49671   \n",
      "\n",
      "     RowNumber  \\\n",
      "124        124   \n",
      "344        344   \n",
      "106        106   \n",
      "110        110   \n",
      "103        103   \n",
      "\n",
      "                                                                  MergedContent  \n",
      "124                                   targeted long term repo operations tltros  \n",
      "344                                   targeted long term repo operations tltros  \n",
      "106   date reserve bank announces third targeted long term repo operation tltro  \n",
      "110       date reserve bank announces targeted long term repo operations tltros  \n",
      "103  date reserve bank announces fourth targeted long term repo operation tltro  \n"
     ]
    }
   ],
   "source": [
    "input_text = \"Special Long-Term Repo Operations (SLTRO) for Small Finance Banks (SFBs)\"\n",
    "similar_documents = find_similar_documents(input_text, cv, vectorized_data, result_df)\n",
    "\n",
    "\n",
    "print(similar_documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1aea5d21",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "58e0a33f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "URL              https://www.rbi.org.in/scripts/FS_Speeches.aspx?Id=1359&fn=6\n",
       "RowNumber                                                                 336\n",
       "MergedContent         date financial sector as an enabler for developed india\n",
       "Name: 336, dtype: object"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.iloc[276]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "10ad48fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                   URL  \\\n",
      "276  https://rbi.org.in/Scripts/BS_PressReleaseDisplay.aspx?prid=38524   \n",
      "\n",
      "     RowNumber  \\\n",
      "276        276   \n",
      "\n",
      "                                                                 MergedContent  \n",
      "276  date rbi issues 500 banknotes inset letter e in mahatma gandhi new series  \n"
     ]
    }
   ],
   "source": [
    "row_number_to_access = 276\n",
    "desired_row = result_df.loc[result_df['RowNumber'] == row_number_to_access]\n",
    "\n",
    "\n",
    "print(desired_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d4f364a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}