03_parent_and_submission.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a CSV with labels, body text, parent text, and score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('datasets/preprocessed_data_2.csv',index_col=None,header=0)\n",
    "data = data[['id','depth','parent_id','score','lemmatized','lemm_sub','labels']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.head()\n",
    "data['labels'][377] = 'denying'\n",
    "data = data.dropna()\n",
    "data = data.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>id</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>lemm_sub</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [index, id, depth, parent_id, score, lemmatized, lemm_sub, labels]\n",
       "Index: []"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "null = data[data.labels.isnull()]\n",
    "null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'no question want thank post topic birth control oft ask one yet meaningful answer not reliably come mainly lack information contribute actionable option'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['lemmatized'][data['id']=='d4xury6'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>id</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>lemm_sub</th>\n",
       "      <th>labels</th>\n",
       "      <th>direct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>d4xury6</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>no question want thank post topic birth contro...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>supporting</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>holy shit thank much someone recently turn sup...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>supporting</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>commenting</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>thanks post tried yaz oral contraceptive prior...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>supporting</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>minor need show parent consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>questioning</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index       id  depth  parent_id  score  \\\n",
       "0      0  d4xury6      0  t3_4r2tl4     31   \n",
       "1      1  d4yi9y9      0  t3_4r2tl4      7   \n",
       "2      2  d4yhm4h      0  t3_4r2tl4      7   \n",
       "3      3  d4y5kmd      0  t3_4r2tl4      3   \n",
       "4      4  d4y8tvk      0  t3_4r2tl4      3   \n",
       "\n",
       "                                          lemmatized  \\\n",
       "0  no question want thank post topic birth contro...   \n",
       "1  holy shit thank much someone recently turn sup...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks post tried yaz oral contraceptive prior...   \n",
       "4                     minor need show parent consent   \n",
       "\n",
       "                                            lemm_sub       labels  \\\n",
       "0  updated september scroll downhi lengthy post c...   supporting   \n",
       "1  updated september scroll downhi lengthy post c...   supporting   \n",
       "2  updated september scroll downhi lengthy post c...   commenting   \n",
       "3  updated september scroll downhi lengthy post c...   supporting   \n",
       "4  updated september scroll downhi lengthy post c...  questioning   \n",
       "\n",
       "                                              direct  \n",
       "0  updated september scroll downhi lengthy post c...  \n",
       "1  updated september scroll downhi lengthy post c...  \n",
       "2  updated september scroll downhi lengthy post c...  \n",
       "3  updated september scroll downhi lengthy post c...  \n",
       "4  updated september scroll downhi lengthy post c...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir_parent = []\n",
    "for i in range(data.shape[0]):\n",
    "    if(data['depth'][i] == 0):\n",
    "        dir_parent.append(data['lemm_sub'][i])\n",
    "    else:\n",
    "        parent_id = data['parent_id'][i][3:]\n",
    "        if( len(data['lemmatized'][data['id']==parent_id]) > 0 ):\n",
    "            dir_parent.append(list(data['lemmatized'][data['id']==parent_id])[0])\n",
    "        else:\n",
    "            dir_parent.append(' ')\n",
    "data['direct'] = dir_parent\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = data[['labels','lemmatized','direct','score']]\n",
    "df.columns = ['labels','body','parent','score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('datasets/labelled_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>body</th>\n",
       "      <th>parent</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>supporting</td>\n",
       "      <td>no question want thank post topic birth contro...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>supporting</td>\n",
       "      <td>holy shit thank much someone recently turn sup...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>commenting</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>supporting</td>\n",
       "      <td>thanks post tried yaz oral contraceptive prior...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>questioning</td>\n",
       "      <td>minor need show parent consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        labels                                               body  \\\n",
       "0   supporting  no question want thank post topic birth contro...   \n",
       "1   supporting  holy shit thank much someone recently turn sup...   \n",
       "2   commenting  people nice friendly super non judgmental espe...   \n",
       "3   supporting  thanks post tried yaz oral contraceptive prior...   \n",
       "4  questioning                     minor need show parent consent   \n",
       "\n",
       "                                              parent  score  \n",
       "0  updated september scroll downhi lengthy post c...     31  \n",
       "1  updated september scroll downhi lengthy post c...      7  \n",
       "2  updated september scroll downhi lengthy post c...      7  \n",
       "3  updated september scroll downhi lengthy post c...      3  \n",
       "4  updated september scroll downhi lengthy post c...      3  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0b4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}