notebooks/intent_zero_shot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "from transformers import ZeroShotClassificationPipeline\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "from tabulate import tabulate\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAX classes per `text`: 3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>label_cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i want to fly from boston at 838 am and arrive...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>what flights are available from pittsburgh to ...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>what is the arrival time in san francisco for ...</td>\n",
       "      <td>flight_time</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cheapest airfare from tacoma to orlando</td>\n",
       "      <td>airfare</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>round trip fares from pittsburgh to philadelph...</td>\n",
       "      <td>airfare</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>845</th>\n",
       "      <td>please find all the flights from cincinnati to...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846</th>\n",
       "      <td>find me a flight from cincinnati to any airpor...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>i 'd like to fly from miami to chicago on amer...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>848</th>\n",
       "      <td>i would like to book a round trip flight from ...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>849</th>\n",
       "      <td>find me a flight that flies from memphis to ta...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5484 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text        label  label_cnt\n",
       "0    i want to fly from boston at 838 am and arrive...       flight          1\n",
       "1    what flights are available from pittsburgh to ...       flight          1\n",
       "2    what is the arrival time in san francisco for ...  flight_time          1\n",
       "3              cheapest airfare from tacoma to orlando      airfare          1\n",
       "4    round trip fares from pittsburgh to philadelph...      airfare          1\n",
       "..                                                 ...          ...        ...\n",
       "845  please find all the flights from cincinnati to...       flight          1\n",
       "846  find me a flight from cincinnati to any airpor...       flight          1\n",
       "847  i 'd like to fly from miami to chicago on amer...       flight          1\n",
       "848  i would like to book a round trip flight from ...       flight          1\n",
       "849  find me a flight that flies from memphis to ta...       flight          1\n",
       "\n",
       "[5484 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.concat([\n",
    "    pd.read_csv('../data/atis/train.tsv', delimiter=\"\\t\", names=['text', 'label']),\n",
    "    pd.read_csv('../data/atis/test.tsv', delimiter=\"\\t\", names=['text', 'label']),\n",
    "])\n",
    "\n",
    "data['label_cnt'] = data['label'].apply(lambda s: len(s.split('+')))\n",
    "print(f\"MAX classes per `text`: {data['label_cnt'].max()}\")\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Median `text` words: 11\n",
      "Max `text` words: 46\n",
      "Max `text` length: 259\n"
     ]
    }
   ],
   "source": [
    "print(f\"Median `text` words: {int(np.median(data['text'].apply(lambda t: len(t.split()))))}\")\n",
    "print(f\"Max `text` words: {np.max(data['text'].apply(lambda t: len(t.split())))}\")\n",
    "print(f\"Max `text` length: {np.max(data['text'].apply(lambda t: len(t)))}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's have a look into `label` values and distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label distribution:\n",
      "label\n",
      "flight                        4039\n",
      "airfare                        451\n",
      "ground_service                 271\n",
      "airline                        176\n",
      "abbreviation                   134\n",
      "aircraft                        86\n",
      "flight_time                     53\n",
      "quantity                        52\n",
      "capacity                        37\n",
      "airport                         31\n",
      "distance                        30\n",
      "flight+airfare                  29\n",
      "ground_fare                     24\n",
      "city                            23\n",
      "flight_no                       20\n",
      "meal                            12\n",
      "restriction                      5\n",
      "day_name                         2\n",
      "airline+flight_no                2\n",
      "airfare+flight_time              1\n",
      "cheapest                         1\n",
      "aircraft+flight+flight_no        1\n",
      "ground_service+ground_fare       1\n",
      "airfare+flight                   1\n",
      "flight+airline                   1\n",
      "flight_no+airline                1\n",
      "Name: count, dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(f\"Label distribution:\\n{data['label'].value_counts()}\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Having entries that look like the following: 'airline+flight_no', 'ground_service+ground_fare', 'aircraft+flight+flight_no' etc., where each of those seems to be build from multiple independent \"base level\" labels, allow us to conclude that we are dealing with a **multiclass classification** case."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's have a look at \"base-level\" labels in both datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abbreviation',\n",
       " 'aircraft',\n",
       " 'airfare',\n",
       " 'airline',\n",
       " 'airport',\n",
       " 'capacity',\n",
       " 'cheapest',\n",
       " 'city',\n",
       " 'day_name',\n",
       " 'distance',\n",
       " 'flight',\n",
       " 'flight_no',\n",
       " 'flight_time',\n",
       " 'ground_fare',\n",
       " 'ground_service',\n",
       " 'meal',\n",
       " 'quantity',\n",
       " 'restriction']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(data['label'].str.split('+').explode().unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choosing a model\n",
    "\n",
    "It appears that trying out a pre-trained ` Zero-Shot Classification`, `Transformers` model from Hugging Face can be a good choice.\n",
    "\n",
    "Using a pre-trained zero-shot model 'raw' without fine-tuning requires careful label preparation to avoid model confusion. To ensure accurate embeddings, we need labels with explicit, simple words, avoiding shortenings or unconventional modifications. Therefore, we replace 'flight_no' with the more explicit 'flight_number' to enhance clarity.\n",
    "\n",
    "Also, to make label tokens more distinct, let's get rid of underscores in the labels too."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abbreviation',\n",
       " 'aircraft',\n",
       " 'airfare',\n",
       " 'airline',\n",
       " 'airport',\n",
       " 'capacity',\n",
       " 'cheapest',\n",
       " 'city',\n",
       " 'day name',\n",
       " 'distance',\n",
       " 'flight',\n",
       " 'flight number',\n",
       " 'flight time',\n",
       " 'ground fare',\n",
       " 'ground service',\n",
       " 'meal',\n",
       " 'quantity',\n",
       " 'restriction']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['label'] = data['label'].str.replace('flight_no', 'flight_number')\n",
    "data['label'] = data['label'].str.replace('_', ' ')\n",
    "\n",
    "# let's have a look at the fupdated labels once again\n",
    "sorted(data['label'].str.split('+').explode().unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see how the text that's labeled as more \"fancier\" categories (that's not present in the `data` DF too) look like in the `test` DF."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For the 'airline+flight number':\n",
      "airline and flight number from columbus to minneapolis\n",
      "\n",
      "For the 'ground service+ground fare':\n",
      "what ground transportation is available from the pittsburgh airport to downtown and how much does it cost\n",
      "\n",
      "For the 'aircraft+flight+flight number':\n",
      "i want to fly from detroit to st. petersburg on northwest airlines and leave around 9 am tell me what aircraft are used by this flight and tell me the flight number\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for l in ['airline+flight number', 'ground service+ground fare', 'aircraft+flight+flight number']:\n",
    "    print(f\"For the '{l}':\\n{data.loc[data['label'] == l].iloc[0]['text']}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def determine_device():\n",
    "    if torch.cuda.is_available():\n",
    "        return torch.device('cuda')\n",
    "    elif torch.backends.mps.is_available():\n",
    "        return torch.device('mps')\n",
    "    else:\n",
    "        return torch.device('cpu')\n",
    "\n",
    "device = determine_device()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing a model on a few \"manual\" inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LABELS len: 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/theflow/.local/share/virtualenvs/intent_classifier-rs8gp2zR/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n",
      "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n",
      "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n",
      "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<transformers.pipelines.zero_shot_classification.ZeroShotClassificationPipeline object at 0x37a905e50>\n",
      "'inference' = {'sequence': 'what is the arrival time in san francisco for the 755 am flight leaving washington', 'labels': ['flight time', 'flight number', 'flight', 'airfare', 'distance', 'airport', 'airline', 'capacity', 'abbreviation', 'aircraft', 'quantity', 'city', 'restriction', 'cheapest', 'ground fare', 'meal', 'ground service'], 'scores': [0.7422202229499817, 0.13428395986557007, 0.0673116073012352, 0.01873125694692135, 0.006247242446988821, 0.005457504186779261, 0.005328348372131586, 0.004026705399155617, 0.0036638150922954082, 0.0036268967669457197, 0.0027513313107192516, 0.001577554503455758, 0.0014490586472675204, 0.001412139623425901, 0.0009985595243051648, 0.000516661733854562, 0.0003971454279962927]}\n",
      "['flight time', 'flight number', 'flight']\n",
      "{'sequence': 'i like my jeans', 'labels': ['fashion', 'distance', 'restriction', 'quantity', 'capacity', 'airfare', 'abbreviation', 'ground fare', 'cheapest', 'city', 'ground service', 'aircraft', 'airline', 'meal', 'flight', 'airport', 'flight number', 'flight time'], 'scores': [0.27146974205970764, 0.20685791969299316, 0.09480439871549606, 0.07539708912372589, 0.06036476045846939, 0.05403900146484375, 0.048942286521196365, 0.041036203503608704, 0.028339853510260582, 0.023743992671370506, 0.02331891469657421, 0.013938872143626213, 0.01383080706000328, 0.011340415105223656, 0.011035689152777195, 0.010274647735059261, 0.0074731409549713135, 0.003792297560721636]}\n",
      "{'sequence': 'little puppies and toddlers are cute', 'labels': ['distance', 'airfare', 'quantity', 'abbreviation', 'city', 'capacity', 'fashion', 'cheapest', 'meal', 'aircraft', 'ground fare', 'restriction', 'ground service', 'flight', 'airport', 'airline', 'flight time', 'flight number'], 'scores': [0.1541144698858261, 0.1354351043701172, 0.0895150676369667, 0.08800243586301804, 0.08126481622457504, 0.061384283006191254, 0.05944505333900452, 0.05232846736907959, 0.048386260867118835, 0.037125445902347565, 0.03437206894159317, 0.03228801488876343, 0.026363959535956383, 0.024222364649176598, 0.022189639508724213, 0.020326193422079086, 0.017175311222672462, 0.01606111228466034]}\n"
     ]
    }
   ],
   "source": [
    "LABELS = [\n",
    "    'abbreviation',\n",
    "    'aircraft',\n",
    "    'airfare',\n",
    "    'airline',\n",
    "    'airport',\n",
    "    'capacity',\n",
    "    'cheapest',\n",
    "    'city',\n",
    "    'distance',\n",
    "    'flight',\n",
    "    'flight number',\n",
    "    'flight time',\n",
    "    'ground fare',\n",
    "    'ground service',\n",
    "    # 'flight_number',\n",
    "    # 'flight_time',\n",
    "    # 'ground_fare',\n",
    "    # 'ground_service',\n",
    "    'meal',\n",
    "    'quantity',\n",
    "    'restriction'\n",
    "]\n",
    "print(f'LABELS len: {len(LABELS)}')\n",
    "\n",
    "model = 'typeform/distilbert-base-uncased-mnli'\n",
    "# model = 'MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli'\n",
    "\n",
    "# Download the model and tokenizer\n",
    "model_name = \"typeform/distilbert-base-uncased-mnli\"\n",
    "model_dir = \"../models\"\n",
    "os.makedirs(model_dir, exist_ok=True)\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_name, cache_dir=model_dir)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=model_dir)\n",
    "\n",
    "classifier_pipeline = ZeroShotClassificationPipeline(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    device=device\n",
    ")\n",
    "\n",
    "print(classifier_pipeline)\n",
    "\n",
    "inference = classifier_pipeline(\n",
    "    # \"i want to fly from detroit to st. petersburg on northwest airlines and leave around 9 am tell me what aircraft are used by this flight and tell me the flight number\",\n",
    "    \"what is the arrival time in san francisco for the 755 am flight leaving washington\",\n",
    "    # \"what day of the week do flights from nashville to tacoma fly on?\",\n",
    "    candidate_labels=LABELS\n",
    ")\n",
    "\n",
    "print(f\"'inference' = {inference}\")\n",
    "\n",
    "def top_k_classes(inference: dict, K: int = 3) -> list:\n",
    "    label_score_pairs = list(zip(inference['labels'], inference['scores']))\n",
    "    # no need to sort as 'labels' come sort by 'score' in descending order in the output\n",
    "    return [pair[0] for pair in label_score_pairs[:K]]\n",
    "\n",
    "\n",
    "print(top_k_classes(inference, 3))\n",
    "\n",
    "# Let's see how the model is reacting to a new extra label and a text on a topic unrelated to ATIS data\n",
    "lbls = LABELS.copy()\n",
    "lbls.append('fashion')\n",
    "inf = classifier_pipeline(\"i like my jeans\", candidate_labels=lbls)\n",
    "print(inf)\n",
    "# the predicted top class 'fashion' looks like a great choice for our test premise ☝️\n",
    "\n",
    "# let's check also the output to an input that's unrelated to our class set from a human POV\n",
    "inf = classifier_pipeline(\"little puppies and toddlers are cute\", candidate_labels=lbls)\n",
    "print(inf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Without replacing underscored to spaces in labels we got:\n",
    "```\n",
    "['flight', 'aircraft', 'flight_number']\n",
    "```\n",
    "as the result for the \n",
    "```\n",
    "\"i want to fly from detroit to st. petersburg on northwest airlines and leave around 9 am tell me what aircraft are used by this flight and tell me the flight number\"\n",
    "```\n",
    "whereas after the pre-processing (fixing wording, replacing underscores with spaces) we got\n",
    "```\n",
    "['flight number', 'flight', 'aircraft']\n",
    "```\n",
    "which appears to be a more accurate outcome (top class 'flight number')."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing\n",
    "\n",
    "As we are not performing any training in this notebook and rather testing a pre-trained zero-shot model's ability to infer intent against our set of labels, we will simply apply the classifier pipline to the whole data set and then calculate and assess metrics."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>label_cnt</th>\n",
       "      <th>inference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i want to fly from boston at 838 am and arrive...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['flight number', 'flight', 'airlin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>what flights are available from pittsburgh to ...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['flight', 'flight number', 'flight...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>what is the arrival time in san francisco for ...</td>\n",
       "      <td>flight time</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['flight time', 'flight number', 'f...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text        label  label_cnt  \\\n",
       "0  i want to fly from boston at 838 am and arrive...       flight          1   \n",
       "1  what flights are available from pittsburgh to ...       flight          1   \n",
       "2  what is the arrival time in san francisco for ...  flight time          1   \n",
       "\n",
       "                                           inference  \n",
       "0  {'labels': ['flight number', 'flight', 'airlin...  \n",
       "1  {'labels': ['flight', 'flight number', 'flight...  \n",
       "2  {'labels': ['flight time', 'flight number', 'f...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's run inference against the data set\n",
    "# (omitting 'sequence' inference's key)\n",
    "data['inference'] = data['text'].apply(lambda t: {k: v for k, v in classifier_pipeline(t, LABELS).items() if k != 'sequence'})\n",
    "data.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Result processing\n",
    "\n",
    "Let's create and populate two new binary (0/1) columns for:\n",
    " * inference's top 3 classes hitting any of the `label` values and \n",
    " * inference's top class hitting  any of the `label` values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>label_cnt</th>\n",
       "      <th>inference</th>\n",
       "      <th>top_3_classes</th>\n",
       "      <th>top_class</th>\n",
       "      <th>top_3_hit</th>\n",
       "      <th>top_hit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i want to fly from boston at 838 am and arrive...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['flight number', 'flight', 'airlin...</td>\n",
       "      <td>[flight number, flight, airline]</td>\n",
       "      <td>flight number</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>what flights are available from pittsburgh to ...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['flight', 'flight number', 'flight...</td>\n",
       "      <td>[flight, flight number, flight time]</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>what is the arrival time in san francisco for ...</td>\n",
       "      <td>flight time</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['flight time', 'flight number', 'f...</td>\n",
       "      <td>[flight time, flight number, flight]</td>\n",
       "      <td>flight time</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text        label  label_cnt  \\\n",
       "0  i want to fly from boston at 838 am and arrive...       flight          1   \n",
       "1  what flights are available from pittsburgh to ...       flight          1   \n",
       "2  what is the arrival time in san francisco for ...  flight time          1   \n",
       "\n",
       "                                           inference  \\\n",
       "0  {'labels': ['flight number', 'flight', 'airlin...   \n",
       "1  {'labels': ['flight', 'flight number', 'flight...   \n",
       "2  {'labels': ['flight time', 'flight number', 'f...   \n",
       "\n",
       "                          top_3_classes      top_class  top_3_hit  top_hit  \n",
       "0      [flight number, flight, airline]  flight number          1        0  \n",
       "1  [flight, flight number, flight time]         flight          1        1  \n",
       "2  [flight time, flight number, flight]    flight time          1        1  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['top_3_classes'] = data['inference'].apply(lambda i: top_k_classes(inference=i, K=3))\n",
    "data['top_class'] = data['inference'].apply(lambda i: top_k_classes(inference=i)[0])\n",
    "\n",
    "# when 'top_3_classes' \"hits\" (has intersections) with any class in the `label` subset - count as a hit\n",
    "data['top_3_hit'] = data.apply(lambda row: int(len(set(row['label'].split('+')) & set(row['top_3_classes'])) > 0), axis=1)\n",
    "data['top_hit'] = data.apply(lambda row: int(row['top_class'] in row['label'].split('+')), axis=1)\n",
    "data.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Performance Metrics\n",
    "\n",
    "Not let's calculate metrics, namely **Precision** and **F1-score**. \n",
    "\n",
    "Note that we don't mention **Recall**. That's because technically speaking we don't have a strictly negative class unlike with binary classification and our labels simply can be farther or closer to the meaning and ontent of a message, but hardly 100% unrelated.\n",
    "\n",
    "With that in mind and knowing that the formula of **Recall** is TP / (TP + FN) and our FN = 0.0 - the concept of **Recall** becomes trivial and we set it to **1.0**.\n",
    "\n",
    "One could argue we could or even should have some sort of a top class threshold, below which the ineference is considered as \"negative\", but we'll leave it outside of the scope of the current exercize for now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------------+---------+\n",
      "| Metric                            |   Value |\n",
      "+===================================+=========+\n",
      "| Total sample size                 |    5484 |\n",
      "+-----------------------------------+---------+\n",
      "| Number of classes                 |      17 |\n",
      "+-----------------------------------+---------+\n",
      "| Top 3 classes inference Precision |    0.85 |\n",
      "+-----------------------------------+---------+\n",
      "| Top 3 classes inference F1        |    0.92 |\n",
      "+-----------------------------------+---------+\n",
      "| Top class inference Precision     |    0.61 |\n",
      "+-----------------------------------+---------+\n",
      "| Top class inference F1            |    0.76 |\n",
      "+-----------------------------------+---------+\n"
     ]
    }
   ],
   "source": [
    "sample_size = data.shape[0]\n",
    "top_3_hits = data['top_3_hit'].sum()\n",
    "top_hits = data['top_hit'].sum()\n",
    "\n",
    "# In our case, by our admission - there are no negative classes, therefore\n",
    "recall = 1.0\n",
    "\n",
    "# Calculate precision, recall, and F1 for top 3 classes\n",
    "top_3_precision = top_3_hits / sample_size\n",
    "top_3_f1 = 2 * (top_3_precision * recall) / (top_3_precision + recall)\n",
    "\n",
    "# Calculate precision, recall, and F1 for top class\n",
    "top_precision = top_hits / sample_size\n",
    "top_f1 = 2 * (top_precision * recall) / (top_precision + recall)\n",
    "\n",
    "headers = [\"Metric\", \"Value\"]\n",
    "report = [\n",
    "    [\"Total sample size\", sample_size],\n",
    "    [\"Number of classes\", len(LABELS)],\n",
    "    [\"Top 3 classes inference Precision\", f\"{top_3_precision:.2f}\"],\n",
    "    [\"Top 3 classes inference F1\", f\"{top_3_f1:.2f}\"],\n",
    "    [\"Top class inference Precision\", f\"{top_precision:.2f}\"],\n",
    "    [\"Top class inference F1\", f\"{top_f1:.2f}\"],\n",
    "]\n",
    "\n",
    "print(tabulate(report, headers, tablefmt=\"grid\", stralign=\"left\", numalign=\"right\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metrics Interpretation\n",
    "\n",
    "#### Top 3 Classes Inference\n",
    "* The model achieves an **F1 score** of **0.92**, indicating excellent performance and high accuracy in identifying message intents.\n",
    "* The **Precision** score of **0.85** is acceptable, with some errors in precision, but still correctly identifying a significant proportion of message intents.\n",
    "\n",
    "#### Top Class Inference\n",
    "* The **F1 score** of **0.76** indicates reasonable accuracy, with some potential errors in either precision or recall.\n",
    "* The **Precision** score of **0.61** is low, with many predicted positive instances being False Positives.\n",
    "\n",
    "Based on these metrics, the \"typeform/distilbert-base-uncased-mnli\" model appears to be a suitable choice for our project's first iteration, as it provides sufficient accuracy in identifying the top 3 intents for a message."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Looking into the \"weakest\" classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>total entries</th>\n",
       "      <th>hits</th>\n",
       "      <th>precision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>day name</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ground fare</td>\n",
       "      <td>25</td>\n",
       "      <td>3</td>\n",
       "      <td>0.120000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>quantity</td>\n",
       "      <td>52</td>\n",
       "      <td>9</td>\n",
       "      <td>0.173077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>airfare</td>\n",
       "      <td>482</td>\n",
       "      <td>177</td>\n",
       "      <td>0.367220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ground service</td>\n",
       "      <td>272</td>\n",
       "      <td>183</td>\n",
       "      <td>0.672794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>flight time</td>\n",
       "      <td>54</td>\n",
       "      <td>41</td>\n",
       "      <td>0.759259</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             label  total entries  hits  precision\n",
       "8         day name              2     0   0.000000\n",
       "13     ground fare             25     3   0.120000\n",
       "16        quantity             52     9   0.173077\n",
       "2          airfare            482   177   0.367220\n",
       "14  ground service            272   183   0.672794\n",
       "12     flight time             54    41   0.759259"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_labels_exploded = data.assign(label=data['label'].str.split('+')).explode('label')\n",
    "label_precision = df_labels_exploded.groupby('label')['top_3_hit'].agg(['count', 'sum']).reset_index()\n",
    "label_precision = label_precision.rename(columns={'sum': 'hits', 'count': 'total entries'})\n",
    "label_precision['precision'] = label_precision['hits'] / label_precision['total entries']\n",
    "label_precision = label_precision.sort_values('precision')\n",
    "\n",
    "label_precision[label_precision['precision'] < 0.8]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looking at the list of the least performing (lowest precisoin) classes, i have an intuition that one thing that can be done to improve the detection of those intents is rephrasing or \"better\" more explicit / accurate wording.\n",
    "\n",
    "For example `day name` can be replaced with `day of the week` etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Investigate the topic of FPs\n",
    "\n",
    "Let's also have a look at \"low-score\" inferences. This might allow us to discover score-thresholds that never lead to a TP inference and can be accepted by us as the model being 'confused' and a case of FP."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|   Score Threshold |   Score <= Threshold, Total Entries |   Top 3 Cls Precision |\n",
      "+===================+=====================================+=======================+\n",
      "|              0    |                                   0 |                  0    |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.02 |                                   0 |                  0    |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.05 |                                   0 |                  0    |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.07 |                                   0 |                  0    |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.09 |                                  15 |                  0.47 |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.11 |                                  91 |                  0.54 |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.14 |                                 217 |                  0.48 |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.16 |                                 327 |                  0.47 |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.18 |                                 425 |                  0.47 |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.2  |                                 527 |                  0.5  |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.23 |                                 626 |                  0.52 |\n",
      "+-------------------+-------------------------------------+-----------------------+\n",
      "|              0.25 |                                 709 |                  0.54 |\n",
      "+-------------------+-------------------------------------+-----------------------+\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>label_cnt</th>\n",
       "      <th>inference</th>\n",
       "      <th>top_3_classes</th>\n",
       "      <th>top_class</th>\n",
       "      <th>top_3_hit</th>\n",
       "      <th>top_hit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i want to fly from boston at 838 am and arrive...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['flight number', 'flight', 'airlin...</td>\n",
       "      <td>[flight number, flight, airline]</td>\n",
       "      <td>flight number</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>show me the first class fares from boston to d...</td>\n",
       "      <td>airfare</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['distance', 'abbreviation', 'fligh...</td>\n",
       "      <td>[distance, abbreviation, flight number]</td>\n",
       "      <td>distance</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>please give me the flights from boston to pitt...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['flight number', 'flight', 'distan...</td>\n",
       "      <td>[flight number, flight, distance]</td>\n",
       "      <td>flight number</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>i would like to fly from denver to pittsburgh ...</td>\n",
       "      <td>flight</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['airline', 'flight', 'aircraft', '...</td>\n",
       "      <td>[airline, flight, aircraft]</td>\n",
       "      <td>airline</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>i 'd like to have some information on a ticket...</td>\n",
       "      <td>airfare</td>\n",
       "      <td>1</td>\n",
       "      <td>{'labels': ['distance', 'airport', 'airfare', ...</td>\n",
       "      <td>[distance, airport, airfare]</td>\n",
       "      <td>distance</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text    label  label_cnt  \\\n",
       "0   i want to fly from boston at 838 am and arrive...   flight          1   \n",
       "15  show me the first class fares from boston to d...  airfare          1   \n",
       "19  please give me the flights from boston to pitt...   flight          1   \n",
       "20  i would like to fly from denver to pittsburgh ...   flight          1   \n",
       "24  i 'd like to have some information on a ticket...  airfare          1   \n",
       "\n",
       "                                            inference  \\\n",
       "0   {'labels': ['flight number', 'flight', 'airlin...   \n",
       "15  {'labels': ['distance', 'abbreviation', 'fligh...   \n",
       "19  {'labels': ['flight number', 'flight', 'distan...   \n",
       "20  {'labels': ['airline', 'flight', 'aircraft', '...   \n",
       "24  {'labels': ['distance', 'airport', 'airfare', ...   \n",
       "\n",
       "                              top_3_classes      top_class  top_3_hit  top_hit  \n",
       "0          [flight number, flight, airline]  flight number          1        0  \n",
       "15  [distance, abbreviation, flight number]       distance          0        0  \n",
       "19        [flight number, flight, distance]  flight number          1        0  \n",
       "20              [airline, flight, aircraft]        airline          1        0  \n",
       "24             [distance, airport, airfare]       distance          1        0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report = []\n",
    "headers = [\"Score Threshold\", \"Score <= Threshold, Total Entries\", \"Top 3 Cls Precision\"]\n",
    "\n",
    "for ls_threshold in np.linspace(0.0, 0.25, 12):\n",
    "    ls_inference = data[[x['scores'][0] <= ls_threshold for x in data['inference']]]\n",
    "    ls_total = ls_inference.shape[0]\n",
    "    hits_cnt = ls_inference['top_3_hit'].sum()\n",
    "    precision = hits_cnt / ls_total if hits_cnt > 0 and ls_total > 0 else 0.0\n",
    "    report.append([f'{ls_threshold:.2f}', ls_total, f'{precision:.2f}'])\n",
    "\n",
    "print(tabulate(report, headers, tablefmt=\"grid\"))\n",
    "\n",
    "ls_inference.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see, there are no entries with a Score less than 0.09 and even at this level we got Precision = 0.47 which is a not bad result considering that we were running inference against 17 classes.\n",
    "\n",
    "With that, accepting Recall as 1.0 seems to be a good call."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ai",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}