metric_twitter2017.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from ast import literal_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metric(data_true, chatgpt_pred):\n",
    "    name_list = [\"person\", \"organization\", \"location\", \"miscellaneous\"]\n",
    "    all_truth_entity,hit_entity,all_pred_entity = 0,0,0\n",
    "    label_entity_all, pred_entity_all = [], []\n",
    "    category_result = {}\n",
    "    entity_notin_sentence = []\n",
    "    for name in name_list:\n",
    "        category_result[name] = {}\n",
    "        category_result[name][\"single_hit\"] = 0\n",
    "        category_result[name][\"single_pred\"] = 0\n",
    "        category_result[name][\"single_true\"] = 0\n",
    "\n",
    "    for i in range(len(chatgpt_pred)):\n",
    "        sentence = chatgpt_pred.loc[i, \"text\"]\n",
    "        for name in name_list:\n",
    "            truth_entity = data_true.loc[i, name]\n",
    "            pred_entity = chatgpt_pred.loc[i, name]\n",
    "            truth_entity_list = literal_eval(truth_entity)\n",
    "            pred_entity_list = literal_eval(pred_entity)\n",
    "            new_truth_list = truth_entity_list\n",
    "            single_hit, single_pred, single_true = 0, 0, 0\n",
    "            if pred_entity_list == ['None']:\n",
    "                new_pred_list = []\n",
    "            else:\n",
    "                new_pred_list = pred_entity_list\n",
    "                \n",
    "            for index in range(len(new_pred_list)):\n",
    "                if str(new_pred_list[index]) not in sentence:\n",
    "                    entity_notin_sentence.append(new_pred_list[index])\n",
    "                else:\n",
    "                    if new_pred_list[index] in new_truth_list:\n",
    "                        hit_entity += 1\n",
    "                        single_hit += 1\n",
    "                    all_pred_entity += 1\n",
    "                    single_pred += 1\n",
    "                    \n",
    "            for index in range(len(new_truth_list)):\n",
    "                if new_truth_list[index] != []:\n",
    "                    all_truth_entity += 1\n",
    "                    single_true += 1\n",
    "            label_entity_all.append(new_truth_list)\n",
    "            pred_entity_all.append(new_pred_list)\n",
    "            category_result[name][\"single_hit\"] += single_hit\n",
    "            category_result[name][\"single_pred\"] += single_pred\n",
    "            category_result[name][\"single_true\"] += single_true\n",
    "\n",
    "    P = hit_entity/all_pred_entity\n",
    "    R = hit_entity/all_truth_entity\n",
    "    F1 = 2*P*R/(P+R)\n",
    "\n",
    "    result = {}\n",
    "    result[\"all_truth_entity\"] = all_truth_entity\n",
    "    result[\"all_pred_entity\"] = all_pred_entity\n",
    "    result[\"hit_entity\"] = hit_entity\n",
    "    result[\"P\"] = P\n",
    "    result[\"R\"] = R\n",
    "    result[\"F1\"] = F1\n",
    "    print(\"entity not in sentence:\", len(entity_notin_sentence))\n",
    "    for name in name_list:\n",
    "        category_result[name][\"P\"] = category_result[name][\"single_hit\"]/category_result[name][\"single_pred\"]\n",
    "        category_result[name][\"R\"] = category_result[name][\"single_hit\"]/category_result[name][\"single_true\"]\n",
    "        category_result[name][\"F1\"] = 2*category_result[name][\"P\"]*category_result[name][\"R\"]/(category_result[name][\"P\"]+category_result[name][\"R\"])\n",
    "    return result, pred_entity_all, label_entity_all, category_result\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "entity not in sentence: 127\n",
      "all_truth_entity: 1351\n",
      "all_pred_entity: 1534\n",
      "hit_entity: 1006\n",
      "P: 0.6558018252933507\n",
      "R: 0.7446336047372317\n",
      "F1: 0.6974003466204506\n",
      "person: {'single_hit': 541, 'single_pred': 588, 'single_true': 621, 'P': 0.9200680272108843, 'R': 0.8711755233494364, 'F1': 0.8949545078577337}\n",
      "organization: {'single_hit': 306, 'single_pred': 529, 'single_true': 395, 'P': 0.5784499054820416, 'R': 0.7746835443037975, 'F1': 0.6623376623376623}\n",
      "location: {'single_hit': 124, 'single_pred': 170, 'single_true': 178, 'P': 0.7294117647058823, 'R': 0.6966292134831461, 'F1': 0.7126436781609194}\n",
      "miscellaneous: {'single_hit': 35, 'single_pred': 247, 'single_true': 157, 'P': 0.1417004048582996, 'R': 0.2229299363057325, 'F1': 0.17326732673267325}\n"
     ]
    }
   ],
   "source": [
    "# multiMM_gpt3.5_twitter2017_10-1_shot-2\n",
    "chatgpt_pred = pd.read_csv(\"multiMM_gpt3.5_twitter2017_50-1_shot-4_predict_test.csv\", sep=\"\\t\")\n",
    "data_true = pd.read_csv(\"./obtain_image_caption/twitter2017_process_caption_test.csv\", sep=\"\\t\")\n",
    "chatgpt_pred = chatgpt_pred.fillna(\"['None']\")\n",
    "result, pred_entity_all, label_entity_all, category_result = metric(data_true, chatgpt_pred)\n",
    "for k in result:\n",
    "    print(k + \":\", result[k])\n",
    "for k in category_result:\n",
    "    print(k + \":\", category_result[k])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ricky",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}