unimodal_Caption.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Caption-based\n",
    "\n",
    "&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;\n",
    "<img align=\"left\" src=\"Images\\unimodal-captions.png\" width=\"600\">"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and set-up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Utils import load_data, preprocessing, model_performances\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import KFold\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 222\n",
    "np.random.seed(seed) # numpy seed\n",
    "tf.random.set_seed(seed) # works for all devices (CPU and GPU)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________Utils ___________________________________________________\n",
    "if not os.path.exists('./Unimodal/predictions'):\n",
    "    os.makedirs('./Unimodal/predictions')\n",
    "\n",
    "if not os.path.exists('./Unimodal/performances'):\n",
    "    os.makedirs('./Unimodal/performances')\n",
    "\n",
    "path_models = './Unimodal/models'\n",
    "file_out = './Unimodal/performances/Captions_results_10Fold.txt'\n",
    "predictions_csv_path = './Unimodal/predictions/Captions_pred_10Fold.csv'\n",
    "\n",
    "file = open(file_out, 'a+')\n",
    "file.truncate(0)  # erase file content\n",
    "file.close()\n",
    "\n",
    "label_column = \"misogynous\"\n",
    "input_columns = ['caption_USE']\n",
    "threshold = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_size = 512  # 512-length array with Universal Sentence Encoder algorithm\n",
    "batch_size = 64\n",
    "epochs = 100"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________load training data ___________________________________________________\n",
    "meme_df = load_data.load_azure_caption_training()\n",
    "\n",
    "meme_df['caption_USE'] = preprocessing.use_preprocessing(meme_df, 'caption')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10 Fold on training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________train model on training data 10Fold________________________________________\n",
    "kf = KFold(n_splits=10, shuffle=False)\n",
    "\n",
    "iteration = 0\n",
    "real_values = np.array([])\n",
    "predict_values = np.array([])\n",
    "ids = np.array([])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for train_index, test_index in kf.split(meme_df):  # split into train and test\n",
    "    preprocessing.set_seed(iteration)\n",
    "    x_train, y_train, x_val, y_val, x_test, y_test = preprocessing.elaborate_data_10fold(meme_df,\n",
    "                                                                                                           train_index,\n",
    "                                                                                                           test_index,\n",
    "                                                                                                           iteration,\n",
    "                                                                                                           input_columns,\n",
    "                                                                                                           label_column)\n",
    "    model, history = model_performances.get_trained_model(x_train, \n",
    "                                                        y_train, \n",
    "                                                        x_val, \n",
    "                                                        y_val,\n",
    "                                                        input_shape=embed_size, \n",
    "                                                        activation_function='LeakyReLU', \n",
    "                                                        neurons=embed_size/2, \n",
    "                                                        dropout=0.2, \n",
    "                                                        epochs=100)\n",
    "    iteration = iteration + 1\n",
    "\n",
    "    # make prediction on training data\n",
    "    pred = model.predict(x_test, batch_size=batch_size)\n",
    "\n",
    "    predict_values = np.append(predict_values, pred)\n",
    "    real_values = np.append(real_values, y_test)\n",
    "    ids = np.append(ids, meme_df.iloc[test_index, :]['file_name'].tolist())\n",
    "\n",
    "    result_df = meme_df.iloc[test_index, [0, 1]]\n",
    "    result_df['score_col'] = pred\n",
    "\n",
    "    # write on file\n",
    "    file = open(file_out, \"a+\")\n",
    "    file.write('\\n\\nITERAZIONE ' + str(iteration) + '\\n')\n",
    "    file.write(json.dumps(model_performances.compute_confusion_rates(result_df, 'score_col', 'misogynous', threshold)))\n",
    "    file.write('\\n') \n",
    "    file.write(classification_report(result_df['misogynous'].values, (result_df['score_col']>threshold).astype(int).values, target_names=['not_mis','mis']))\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# results dataframe, save predictions\n",
    "result_df = pd.DataFrame({'id': ids, 'real': real_values.astype(int), 'pred': predict_values})\n",
    "result_df.to_csv(predictions_csv_path, index=False, sep='\\t')\n",
    "\n",
    "# Overall metrics _ write on file\n",
    "file = open(file_out, \"a+\")\n",
    "file.write('\\n\\n10 Fold Results ' + str(iteration) + '\\n')\n",
    "file.write(json.dumps(model_performances.compute_confusion_rates(result_df, 'pred', 'real', threshold)))\n",
    "file.write('\\n') \n",
    "file.write(classification_report(result_df['real'].values, (result_df['pred']>threshold).astype(int).values, target_names=['not_mis','mis']))\n",
    "file.write('\\n AUC:') \n",
    "file.write(str(model_performances.compute_auc(result_df['real'].values, result_df['pred'].values)))\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# summarize history for accuracy\n",
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'dev'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'dev'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10 Fold on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________Utils ___________________________________________________\n",
    "if not os.path.exists('./Unimodal/predictionsTest'):\n",
    "    os.makedirs('./Unimodal/predictionsTest')\n",
    "\n",
    "if not os.path.exists('./Unimodal/performancesTest'):\n",
    "    os.makedirs('./Unimodal/performancesTest')\n",
    "\n",
    "path_models = './Unimodal/modelsTest'\n",
    "file_out = './Unimodal/performancesTest/Captions_results_10Fold.txt'\n",
    "predictions_csv_path = './Unimodal/predictionsTest/Captions_pred_10Fold.csv'\n",
    "\n",
    "file = open(file_out, 'a+')\n",
    "file.truncate(0)  \n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Test and preprocessing\n",
    "test_df = load_data.load_azure_caption_test()\n",
    "test_df['caption_USE'] = preprocessing.use_preprocessing(test_df, 'caption')\n",
    "\n",
    "x_test, y_test = preprocessing.elaborate_input(test_df, input_columns, label_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________train model on training data 10Fold________________________________________\n",
    "kf = KFold(n_splits=10, shuffle=False)\n",
    "\n",
    "iteration = 0\n",
    "real_values = np.array([])\n",
    "predict_values = np.array([])\n",
    "ids = np.array([])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for train_index, val_index in kf.split(meme_df):  # split into train and test\n",
    "    preprocessing.set_seed(iteration)\n",
    "    x_train, y_train = preprocessing.elaborate_input(meme_df.iloc[train_index, :], input_columns, label_column)\n",
    "    x_val, y_val = preprocessing.elaborate_input(meme_df.iloc[val_index, :], input_columns, label_column)\n",
    "\n",
    "    model, history = model_performances.get_trained_model(x_train, \n",
    "                            y_train, \n",
    "                            x_val, \n",
    "                            y_val,\n",
    "                            input_shape=embed_size, \n",
    "                            activation_function='LeakyReLU', \n",
    "                            neurons=embed_size/2, \n",
    "                            dropout=0.2, \n",
    "                            epochs=100)\n",
    "\n",
    "    iteration = iteration + 1\n",
    "\n",
    "    # make prediction on training data\n",
    "    pred = model.predict(x_test, batch_size=batch_size)\n",
    "\n",
    "    predict_values = np.append(predict_values, pred)\n",
    "    real_values = np.append(real_values, y_test)\n",
    "    ids = np.append(ids, test_df['file_name'].tolist())\n",
    "\n",
    "    result_df = test_df[['file_name', 'misogynous']].copy()\n",
    "    result_df['score_col'] = pred\n",
    "    \n",
    "    # write on file\n",
    "    file = open(file_out, \"a+\")\n",
    "    file.write('\\n\\nITERAZIONE ' + str(iteration) + '\\n')\n",
    "    file.write(json.dumps(model_performances.compute_confusion_rates(result_df, 'score_col', 'misogynous', threshold)))\n",
    "    file.write('\\n') \n",
    "    file.write(classification_report(result_df['misogynous'].values, (result_df['score_col']>threshold).astype(int).values, target_names=['not_mis','mis']))\n",
    "    file.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# summarize history for accuracy\n",
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'dev'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'dev'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tpr': 0.8988,\n",
       " 'tnr': 0.2118,\n",
       " 'fpr': 0.7882,\n",
       " 'fnr': 0.10119999999999996,\n",
       " 'precision': 0.5327800829875519,\n",
       " 'recall': 0.8988,\n",
       " 'accuracy': 0.5553,\n",
       " 'f1': 0.6689988835132118,\n",
       " 'auc': 0.6087016}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# results dataframe, save predictions\n",
    "result_df = pd.DataFrame({'id': ids, 'real': real_values.astype(int), 'pred': predict_values})\n",
    "result_df.to_csv(predictions_csv_path, index=False, sep='\\t')\n",
    "\n",
    "# Overall metrics _ write on file\n",
    "file = open(file_out, \"a+\")\n",
    "file.write('\\n\\n10 Fold Results ' + str(iteration) + '\\n')\n",
    "file.write(json.dumps(model_performances.compute_confusion_rates(result_df, 'pred', 'real', threshold)))\n",
    "file.write('\\n') \n",
    "file.write(classification_report(result_df['real'].values, (result_df['pred']>threshold).astype(int).values, target_names=['not_mis','mis']))\n",
    "file.write('\\n AUC:') \n",
    "file.write(str(model_performances.compute_auc(result_df['real'].values, result_df['pred'].values)))\n",
    "file.close()\n",
    "model_performances.compute_confusion_rates(result_df, 'pred', 'real', threshold)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0 (tags/v3.10.0:b494f59, Oct  4 2021, 19:00:18) [MSC v.1929 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d1254c3cc9e57f52b804b6ab65bbeea341813290580c92b29841c38c0e3519f7"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}